From 8831db281db73a3a4c119889bcfbbea86ea7730c Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 14 May 2025 12:59:56 +0400 Subject: [PATCH 01/55] initial draft of market 2.0 --- cmd/curio/tasks/tasks.go | 8 +- cuhttp/server.go | 13 +- deps/config/doc_gen.go | 53 + deps/config/types.go | 36 + .../default-curio-configuration.md | 39 + go.mod | 2 + go.sum | 4 + .../harmonydb/sql/20250505-market_mk20.sql | 93 ++ market/http/http.go | 19 +- market/mk12/http/http.go | 2 +- market/mk12/mk12.go | 2 +- market/mk20/ddo_v1.go | 163 +++ market/mk20/http/http.go | 165 +++ market/mk20/mk20.go | 291 +++++ market/mk20/types.go | 154 +++ market/mk20/utils.go | 450 +++++++ market/retrieval/piecehandler.go | 12 +- tasks/indexing/task_indexing.go | 190 ++- tasks/indexing/task_ipni.go | 67 +- tasks/seal/poller_commit_msg.go | 9 +- tasks/seal/task_movestorage.go | 12 +- tasks/snap/task_movestorage.go | 12 +- tasks/snap/task_submit.go | 8 +- tasks/storage-market/mk20.go | 1085 +++++++++++++++++ tasks/storage-market/storage_market.go | 41 +- tasks/storage-market/task_aggregation.go | 344 ++++++ tasks/storage-market/task_commp.go | 145 ++- 27 files changed, 3284 insertions(+), 135 deletions(-) create mode 100644 harmony/harmonydb/sql/20250505-market_mk20.sql create mode 100644 market/mk20/ddo_v1.go create mode 100644 market/mk20/http/http.go create mode 100644 market/mk20/mk20.go create mode 100644 market/mk20/types.go create mode 100644 market/mk20/utils.go create mode 100644 tasks/storage-market/mk20.go create mode 100644 tasks/storage-market/task_aggregation.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 22219046c..f70d8b082 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -244,16 +244,19 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan } { + var sdeps cuhttp.ServiceDeps // Market tasks var dm *storage_market.CurioStorageDealMarket if cfg.Subsystems.EnableDealMarket { // Main market poller should run on all nodes - dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, si, full, as) + dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as) err := dm.StartMarket(ctx) if err != nil { return nil, err } + sdeps.DealMarket = dm + if cfg.Subsystems.EnableCommP { commpTask := storage_market.NewCommpTask(dm, db, must.One(slrLazy.Val()), full, cfg.Subsystems.CommPMaxTasks) activeTasks = append(activeTasks, commpTask) @@ -275,7 +278,6 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan if err != nil { return nil, err } - var sdeps cuhttp.ServiceDeps if cfg.Subsystems.EnablePDP { es := getSenderEth() @@ -298,7 +300,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan activeTasks = append(activeTasks, ipniTask, indexingTask) if cfg.HTTP.Enable { - err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps, dm) + err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps) if err != nil { return nil, xerrors.Errorf("failed to start the HTTP server: %w", err) } diff --git a/cuhttp/server.go b/cuhttp/server.go index 4a5090367..716da25a7 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -136,10 +136,11 @@ func isWebSocketUpgrade(r *http.Request) bool { } type ServiceDeps struct { - EthSender *message.SenderETH + EthSender *message.SenderETH + DealMarket *storage_market.CurioStorageDealMarket } -func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps, dm *storage_market.CurioStorageDealMarket) error { +func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { cfg := d.Cfg.HTTP // Setup the Chi router for more complex routing (if needed in the future) @@ -181,7 +182,9 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps, dm *sto fmt.Fprintf(w, "Service is up and running") }) - chiRouter, err = attachRouters(ctx, chiRouter, d, sd, dm) + // TODO: Attach a info page here with details about all the service and endpoints + + chiRouter, err = attachRouters(ctx, chiRouter, d, sd) if err != nil { return xerrors.Errorf("failed to attach routers: %w", err) } @@ -275,7 +278,7 @@ func (c cache) Delete(ctx context.Context, key string) error { var _ autocert.Cache = cache{} -func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDeps, dm *storage_market.CurioStorageDealMarket) (*chi.Mux, error) { +func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDeps) (*chi.Mux, error) { // Attach retrievals rp := retrieval.NewRetrievalProvider(ctx, d.DB, d.IndexStore, d.CachedPieceReader) retrieval.Router(r, rp) @@ -299,7 +302,7 @@ func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDep } // Attach the market handler - dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, dm) + dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket) if err != nil { return nil, xerrors.Errorf("failed to create new market handler: %w", err) } diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 7a2b85588..6e0fa47e8 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -1045,6 +1045,53 @@ CIDGravity filters will not be applied to deals associated with that miner ID.`, Default behaviors is to reject the deals (Default: false)`, }, }, + "MK20Config": { + { + Name: "ExpectedPoRepSealDuration", + Type: "time.Duration", + + Comment: `ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector +This will be used to fail the deals which cannot be sealed on time. +Time duration string (e.g., "1h2m3s") in TOML format. (Default: "8h0m0s")`, + }, + { + Name: "ExpectedSnapSealDuration", + Type: "time.Duration", + + Comment: `ExpectedSnapSealDuration is the expected time it would take to snap the deal sector +This will be used to fail the deals which cannot be sealed on time. +Time duration string (e.g., "1h2m3s") in TOML format. (Default: "2h0m0s")`, + }, + { + Name: "SkipCommP", + Type: "bool", + + Comment: `SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain +Warning: If this check is skipped and there is a commP mismatch, all deals in the +sector will need to be sent again (Default: false)`, + }, + { + Name: "DisabledMiners", + Type: "[]string", + + Comment: `DisabledMiners is a list of miner addresses that should be excluded from online deal making protocols`, + }, + { + Name: "MaxConcurrentDealSizeGiB", + Type: "int64", + + Comment: `MaxConcurrentDealSizeGiB is a sum of all size of all deals which are waiting to be added to a sector +When the cumulative size of all deals in process reaches this number, new deals will be rejected. +(Default: 0 = unlimited)`, + }, + { + Name: "DenyUnknownClients", + Type: "bool", + + Comment: `DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list +If True then all deals coming from unknown clients will be rejected. (Default: false)`, + }, + }, "MarketConfig": { { Name: "StorageMarketConfig", @@ -1149,6 +1196,12 @@ Example: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXX Comment: `MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals)`, }, + { + Name: "MK20", + Type: "MK20Config", + + Comment: `MK20 encompasses all configuration related to deal protocol mk2.0 i.e. market 2.0`, + }, { Name: "IPNI", Type: "IPNIConfig", diff --git a/deps/config/types.go b/deps/config/types.go index c856e58c6..a7ab451d5 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -110,6 +110,10 @@ func DefaultCurioConfig() *CurioConfig { ExpectedSnapSealDuration: 2 * time.Hour, CIDGravityTokens: []string{}, }, + MK20: MK20Config{ + ExpectedPoRepSealDuration: 8 * time.Hour, + ExpectedSnapSealDuration: 2 * time.Hour, + }, IPNI: IPNIConfig{ ServiceURL: []string{"https://cid.contact"}, DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"}, @@ -693,6 +697,9 @@ type StorageMarketConfig struct { // MK12 encompasses all configuration related to deal protocol mk1.2.0 and mk1.2.1 (i.e. Boost deals) MK12 MK12Config + // MK20 encompasses all configuration related to deal protocol mk2.0 i.e. market 2.0 + MK20 MK20Config + // IPNI configuration for ipni-provider IPNI IPNIConfig @@ -873,3 +880,32 @@ type MK12CollateralConfig struct { // Accepts a decimal string (e.g., "123.45" or "123 fil") with optional "fil" or "attofil" suffix. (Default: "20 FIL") CollateralHighThreshold types.FIL } + +type MK20Config struct { + // ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector + // This will be used to fail the deals which cannot be sealed on time. + // Time duration string (e.g., "1h2m3s") in TOML format. (Default: "8h0m0s") + ExpectedPoRepSealDuration time.Duration + + // ExpectedSnapSealDuration is the expected time it would take to snap the deal sector + // This will be used to fail the deals which cannot be sealed on time. + // Time duration string (e.g., "1h2m3s") in TOML format. (Default: "2h0m0s") + ExpectedSnapSealDuration time.Duration + + // SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain + // Warning: If this check is skipped and there is a commP mismatch, all deals in the + // sector will need to be sent again (Default: false) + SkipCommP bool + + // DisabledMiners is a list of miner addresses that should be excluded from online deal making protocols + DisabledMiners []string + + // MaxConcurrentDealSizeGiB is a sum of all size of all deals which are waiting to be added to a sector + // When the cumulative size of all deals in process reaches this number, new deals will be rejected. + // (Default: 0 = unlimited) + MaxConcurrentDealSizeGiB int64 + + // DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list + // If True then all deals coming from unknown clients will be rejected. (Default: false) + DenyUnknownClients bool +} diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 2a6a7e29e..aa07c553f 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -660,6 +660,45 @@ description: The default curio configuration # type: bool #DefaultCIDGravityAccept = false + # MK20 encompasses all configuration related to deal protocol mk2.0 i.e. market 2.0 + # + # type: MK20Config + [Market.StorageMarketConfig.MK20] + + # ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector + # This will be used to fail the deals which cannot be sealed on time. + # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "8h0m0s") + # + # type: time.Duration + #ExpectedPoRepSealDuration = "8h0m0s" + + # ExpectedSnapSealDuration is the expected time it would take to snap the deal sector + # This will be used to fail the deals which cannot be sealed on time. + # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "2h0m0s") + # + # type: time.Duration + #ExpectedSnapSealDuration = "2h0m0s" + + # SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain + # Warning: If this check is skipped and there is a commP mismatch, all deals in the + # sector will need to be sent again (Default: false) + # + # type: bool + #SkipCommP = false + + # MaxConcurrentDealSizeGiB is a sum of all size of all deals which are waiting to be added to a sector + # When the cumulative size of all deals in process reaches this number, new deals will be rejected. + # (Default: 0 = unlimited) + # + # type: int64 + #MaxConcurrentDealSizeGiB = 0 + + # DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list + # If True then all deals coming from unknown clients will be rejected. (Default: false) + # + # type: bool + #DenyUnknownClients = false + # IPNI configuration for ipni-provider # # type: IPNIConfig diff --git a/go.mod b/go.mod index 44d710aa2..5cee3cb6d 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/filecoin-project/go-commp-utils v0.1.4 github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 + github.com/filecoin-project/go-data-segment v0.0.1 github.com/filecoin-project/go-f3 v0.8.4 github.com/filecoin-project/go-fil-commcid v0.2.0 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 @@ -84,6 +85,7 @@ require ( github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-varint v0.0.7 + github.com/oklog/ulid v1.3.1 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.20.5 diff --git a/go.sum b/go.sum index 2063a0ee1..beff2c37e 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= +github.com/filecoin-project/go-data-segment v0.0.1 h1:1wmDxOG4ubWQm3ZC1XI5nCon5qgSq7Ra3Rb6Dbu10Gs= +github.com/filecoin-project/go-data-segment v0.0.1/go.mod h1:H0/NKbsRxmRFBcLibmABv+yFNHdmtl5AyplYLnb0Zv4= github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t8tbt1Hz9qENnN4nVr1w99aSWc= github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE= github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w= @@ -1105,6 +1107,8 @@ github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOW github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql new file mode 100644 index 000000000..67e08b85b --- /dev/null +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -0,0 +1,93 @@ +CREATE TABLE ddo_contracts ( + address TEXT NOT NULL PRIMARY KEY, + abi TEXT NOT NULL +); + +CREATE TABLE market_mk20_deal ( + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + id TEXT PRIMARY KEY, + piece_cid TEXT NOT NULL, + size BIGINT NOT NULL, + + format JSONB NOT NULL, + source_http JSONB NOT NULL DEFAULT 'null', + source_aggregate JSONB NOT NULL DEFAULT 'null', + source_offline JSONB NOT NULL DEFAULT 'null', + + ddov1 JSONB NOT NULL DEFAULT 'null', + market_deal_id TEXT DEFAULT NULL +); + +CREATE TABLE market_mk20_pipeline ( + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + id TEXT NOT NULL, + sp_id BIGINT NOT NULL, + contract TEXT NOT NULL, + client TEXT NOT NULL, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + raw_size BIGINT NOT NULL, + offline BOOLEAN NOT NULL, + url TEXT DEFAULT NULL, + indexing BOOLEAN NOT NULL, + announce BOOLEAN NOT NULL, + allocation_id BIGINT DEFAULT NULL, + duration BIGINT NOT NULL, + piece_aggregation INT DEFAULT 0, + + started BOOLEAN DEFAULT FALSE, + + downloaded BOOLEAN DEFAULT FALSE, + + commp_task_id BIGINT DEFAULT NULL, + after_commp BOOLEAN DEFAULT FALSE, + + deal_aggregation INT DEFAULT 0, + aggr_index BIGINT DEFAULT 0, + agg_task_id BIGINT DEFAULT NULL, + aggregated BOOLEAN DEFAULT FALSE, + + sector BIGINT DEFAULT NULL, + reg_seal_proof INT DEFAULT NULL, + sector_offset BIGINT DEFAULT NULL, -- padded offset + + sealed BOOLEAN DEFAULT FALSE, + + indexing_created_at TIMESTAMPTZ DEFAULT NULL, + indexing_task_id BIGINT DEFAULT NULL, + indexed BOOLEAN DEFAULT FALSE, + + complete BOOLEAN NOT NULL DEFAULT FALSE, + + PRIMARY KEY (id, aggr_index) +); + +CREATE TABLE market_mk20_pipeline_waiting ( + id TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE market_mk20_download_pipeline ( + id TEXT NOT NULL, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + ref_ids BIGINT[] NOT NULL, + PRIMARY KEY (id, piece_cid, piece_size) +); + +CREATE TABLE market_mk20_offline_urls ( + id TEXT NOT NULL, + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + url TEXT NOT NULL, + headers jsonb NOT NULL DEFAULT '{}', + raw_size BIGINT NOT NULL, + CONSTRAINT market_mk20_offline_urls_id_fk FOREIGN KEY (id) + REFERENCES market_mk20_deal_pipeline (id) + ON DELETE CASCADE, + CONSTRAINT market_mk20_offline_urls_id_unique UNIQUE (id) +); + + + + + diff --git a/market/http/http.go b/market/http/http.go index a9c4934bf..b4fb23803 100644 --- a/market/http/http.go +++ b/market/http/http.go @@ -6,27 +6,38 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" mk12http "github.com/filecoin-project/curio/market/mk12/http" + mk20http "github.com/filecoin-project/curio/market/mk20/http" storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) type MarketHandler struct { - mdh *mk12http.MK12DealHandler + mdh12 *mk12http.MK12DealHandler + mdh20 *mk20http.MK20DealHandler } // NewMarketHandler is used to prepare all the required market handlers. Currently, it supports mk12 deal market. // This function should be used to expand the functionality under "/market" path func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_market.CurioStorageDealMarket) (*MarketHandler, error) { - mdh, err := mk12http.NewMK12DealHandler(db, cfg, dm) + mdh12, err := mk12http.NewMK12DealHandler(db, cfg, dm) if err != nil { return nil, err } + + mdh20, err := mk20http.NewMK20DealHandler(db, cfg, dm) + if err != nil { + return nil, err + } + return &MarketHandler{ - mdh: mdh, + mdh12: mdh12, + mdh20: mdh20, }, nil } // Router is used to attach all the market handlers // This can include mk12 deals, mk20 deals(WIP), sector market(WIP) etc func Router(mux *chi.Mux, mh *MarketHandler) { - mux.Mount("/market/mk12", mk12http.Router(mh.mdh)) + mux.Mount("/market/mk12", mk12http.Router(mh.mdh12)) + mux.Mount("/market/mk20", mk20http.Router(mh.mdh20)) + // TODO: Attach a info endpoint here with details about supported market modules and services under them } diff --git a/market/mk12/http/http.go b/market/mk12/http/http.go index eff6a15a7..cf478f4c6 100644 --- a/market/mk12/http/http.go +++ b/market/mk12/http/http.go @@ -22,7 +22,7 @@ import ( storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) -var log = logging.Logger("mktdealhdlr") +var log = logging.Logger("mk12httphdlr") // Redirector struct with a database connection type MK12DealHandler struct { diff --git a/market/mk12/mk12.go b/market/mk12/mk12.go index 67ca50e71..3ac6b28d0 100644 --- a/market/mk12/mk12.go +++ b/market/mk12/mk12.go @@ -561,7 +561,7 @@ func (m *MK12) processDeal(ctx context.Context, deal *ProviderDealState) (*Provi if !deal.IsOffline { var pieceID int64 // Attempt to select the piece ID first - err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1`, prop.PieceCID.String()).Scan(&pieceID) + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, prop.PieceCID.String(), prop.PieceSize).Scan(&pieceID) if err != nil { if errors.Is(err, pgx.ErrNoRows) { diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go new file mode 100644 index 000000000..b1b4a0633 --- /dev/null +++ b/market/mk20/ddo_v1.go @@ -0,0 +1,163 @@ +package mk20 + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/ethereum/go-ethereum" + eabi "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +var UnknowContract = errors.New("provider does not work with this market") + +// DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details +// for a DDO deal handling. +type DDOV1 struct { + + // Provider specifies the address of the provider + Provider address.Address `json:"provider"` + + // Client represents the address of the deal client + Client address.Address `json:"client"` + + // Actor able to with AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs + PieceManager address.Address `json:"piecemanager"` + + // Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. + // It must be at least 518400 + Duration abi.ChainEpoch `json:"duration"` + + // AllocationId represents an aggregated allocation identifier for the deal. + AllocationId *verifreg.AllocationId `json:"aggregatedallocationid"` + + // ContractAddress specifies the address of the contract governing the deal + ContractAddress string `json:"contractaddress"` + + // ContractDealIDMethod specifies the method name to retrieve the deal ID for a contract + ContractDealIDMethod string `json:"contractdealidmethod"` + + // ContractDealIDMethodParams represents encoded parameters for the contract deal ID method if required by the contract + ContractDealIDMethodParams []byte `json:"contractdealidmethodparams"` + + // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated + NotificationAddress string `json:"notificationaddress"` + + // NotificationPayload holds the notification data typically in a serialized byte array format. + NotificationPayload []byte `json:"notificationpayload"` + + // Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval + Indexing bool `json:"indexing"` + + // AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). + AnnounceToIPNI bool `json:"announcetoinpni"` +} + +func (d *DDOV1) Validate() error { + if d.Provider == address.Undef || d.Provider.Empty() { + return xerrors.Errorf("provider address is not set") + } + + if d.Client == address.Undef || d.Client.Empty() { + return xerrors.Errorf("client address is not set") + } + + if d.PieceManager == address.Undef || d.PieceManager.Empty() { + return xerrors.Errorf("piece manager address is not set") + } + + if d.AllocationId != nil { + if *d.AllocationId == verifreg.NoAllocationID { + return xerrors.Errorf("incorrect allocation id") + } + } + + if d.AllocationId == nil { + if d.Duration < 518400 { + return xerrors.Errorf("duration must be at least 518400") + } + } + + if d.ContractAddress == "" { + return xerrors.Errorf("contract address is not set") + } + + if d.ContractAddress[0:2] != "0x" { + return xerrors.Errorf("contract address must start with 0x") + } + + if d.ContractDealIDMethodParams == nil { + return xerrors.Errorf("contract deal id method params is not set") + } + + if d.ContractDealIDMethod == "" { + return xerrors.Errorf("contract deal id method is not set") + } + + return nil +} + +func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, error) { + var abiStr string + err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", UnknowContract + } + return "", xerrors.Errorf("getting abi: %w", err) + } + + parsedABI, err := eabi.JSON(strings.NewReader(abiStr)) + if err != nil { + panic(err) + } + + to := common.HexToAddress(d.ContractAddress) + + // Get the method + method, exists := parsedABI.Methods[d.ContractDealIDMethod] + if !exists { + return "", fmt.Errorf("method %s not found in ABI", d.ContractDealIDMethod) + } + + // Enforce method must take exactly one `bytes` parameter + if len(method.Inputs) != 1 || method.Inputs[0].Type.String() != "bytes" { + return "", fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) + } + + // ABI-encode method call with input + callData, err := parsedABI.Pack(method.Name, d.ContractDealIDMethodParams) + if err != nil { + return "", fmt.Errorf("failed to encode call data: %w", err) + } + + // Build call message + msg := ethereum.CallMsg{ + To: &to, + Data: callData, + } + + // Call contract + output, err := eth.CallContract(ctx, msg, nil) + if err != nil { + return "", fmt.Errorf("eth_call failed: %w", err) + } + + // Decode return value (assume string) + var result string + if err := parsedABI.UnpackIntoInterface(&result, method.Name, output); err != nil { + return "", fmt.Errorf("decode result: %w", err) + } + return result, nil +} diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go new file mode 100644 index 000000000..eea888af6 --- /dev/null +++ b/market/mk20/http/http.go @@ -0,0 +1,165 @@ +package http + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/httprate" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/market/mk20" + storage_market "github.com/filecoin-project/curio/tasks/storage-market" +) + +var log = logging.Logger("mk20httphdlr") + +type MK20DealHandler struct { + cfg *config.CurioConfig + db *harmonydb.DB // Replace with your actual DB wrapper if different + dm *storage_market.CurioStorageDealMarket + disabledMiners []address.Address +} + +func NewMK20DealHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_market.CurioStorageDealMarket) (*MK20DealHandler, error) { + var disabledMiners []address.Address + for _, m := range cfg.Market.StorageMarketConfig.MK12.DisabledMiners { + maddr, err := address.NewFromString(m) + if err != nil { + return nil, xerrors.Errorf("failed to parse miner string: %s", err) + } + disabledMiners = append(disabledMiners, maddr) + } + return &MK20DealHandler{db: db, dm: dm, cfg: cfg, disabledMiners: disabledMiners}, nil +} + +func dealRateLimitMiddleware() func(http.Handler) http.Handler { + return httprate.LimitByIP(50, 1*time.Second) +} + +func Router(mdh *MK20DealHandler) http.Handler { + mux := chi.NewRouter() + mux.Use(dealRateLimitMiddleware()) + mux.Post("/store", mdh.mk20deal) + //mux.Get("/ask", mdh.mk20ask) + mux.Get("/status", mdh.mk20status) + mux.Get("/contracts", mdh.mk20supportedContracts) + return mux +} + +func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { + ct := r.Header.Get("Content-Type") + var deal mk20.Deal + if ct != "application/json" { + log.Errorf("invalid content type: %s", ct) + w.WriteHeader(http.StatusBadRequest) + return + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("error reading request body: %s", err) + w.WriteHeader(http.StatusBadRequest) + } + err = json.Unmarshal(body, &deal) + if err != nil { + log.Errorf("error unmarshaling json: %s", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + result := mdh.dm.MK20Handler.ExecuteDeal(context.Background(), &deal) + + log.Infow("deal processed", + "id", deal.Identifier, + "HTTPCode", result.HTTPCode, + "Reason", result.Reason) + + w.WriteHeader(result.HTTPCode) + _, err = w.Write([]byte(fmt.Sprint("Reason: ", result.Reason))) + if err != nil { + log.Errorw("writing deal response:", "id", deal.Identifier, "error", err) + } +} + +func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { + ct := r.Header.Get("Content-Type") + var request mk20.DealStatusRequest + + if ct != "application/json" { + log.Errorf("invalid content type: %s", ct) + w.WriteHeader(http.StatusBadRequest) + return + } + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("error reading request body: %s", err) + w.WriteHeader(http.StatusBadRequest) + } + err = json.Unmarshal(body, &request) + if err != nil { + log.Errorf("error unmarshaling json: %s", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + result, err := mdh.dm.MK20Handler.DealStatus(context.Background(), &request) + if err != nil { + log.Errorw("failed to get deal status", "id", request.Identifier, + "idType", request.IdentifierType, + "contractAddress", request.ContractAddress, "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + resp, err := json.Marshal(result) + if err != nil { + log.Errorw("failed to marshal deal status response", "id", request.Identifier, + "idType", request.IdentifierType, + "contractAddress", request.ContractAddress, "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(resp) + if err != nil { + log.Errorw("failed to write deal status response", "id", request.Identifier, + "idType", request.IdentifierType, + "contractAddress", request.ContractAddress, "err", err) + } +} + +func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *http.Request) { + var contracts mk20.SupportedContracts + err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM contracts") + if err != nil { + log.Errorw("failed to get supported contracts", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + // Write a json array + resp, err := json.Marshal(contracts) + if err != nil { + log.Errorw("failed to marshal supported contracts", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(resp) + if err != nil { + log.Errorw("failed to write supported contracts", "err", err) + } +} diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go new file mode 100644 index 000000000..918c47e66 --- /dev/null +++ b/market/mk20/mk20.go @@ -0,0 +1,291 @@ +package mk20 + +import ( + "context" + "fmt" + "net/http" + + "github.com/ethereum/go-ethereum/ethclient" + logging "github.com/ipfs/go-log/v2" + "github.com/samber/lo" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v16/miner" + "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + verifreg9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/multictladdr" + "github.com/filecoin-project/curio/lib/paths" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/types" +) + +var log = logging.Logger("mk20") + +type MK20API interface { + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) + StateGetAllocation(ctx context.Context, clientAddr address.Address, allocationId verifreg9.AllocationId, tsk types.TipSetKey) (*verifreg9.Allocation, error) +} + +type MK20 struct { + miners []address.Address + db *harmonydb.DB + api MK20API + ethClient *ethclient.Client + si paths.SectorIndex + cfg *config.CurioConfig + sm map[address.Address]abi.SectorSize + as *multictladdr.MultiAddressSelector +} + +func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector) (*MK20, error) { + ctx := context.Background() + + sm := make(map[address.Address]abi.SectorSize) + + for _, m := range miners { + info, err := mapi.StateMinerInfo(ctx, m, types.EmptyTSK) + if err != nil { + return nil, xerrors.Errorf("getting miner info: %w", err) + } + if _, ok := sm[m]; !ok { + sm[m] = info.SectorSize + } + } + + return &MK20{ + miners: miners, + db: db, + api: mapi, + ethClient: ethClient, + si: si, + cfg: cfg, + sm: sm, + as: as, + }, nil +} + +func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { + // Validate the DataSource TODO: Add error code to validate + valid, err := deal.Validate() + if err != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Invalid data source", + } + } + if !valid { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Invalid data source", + } + } + + return m.processDDODeal(ctx, deal) + +} + +func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { + rejection, err := m.sanitizeDDODeal(ctx, deal) + if err != nil { + log.Errorw("deal rejected", "deal", deal, "error", err) + return rejection + } + if rejection != nil { + return rejection + } + + id, err := deal.Products.DDOV1.GetDealID(ctx, m.db, m.ethClient) + if err != nil { + log.Errorw("error getting deal ID", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + + // TODO: Backpressure, client filter + + comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = deal.SaveToDB(tx) + if err != nil { + return false, err + } + n, err := tx.Exec(`Update market_mk20_deal SET market_deal_id = $1 WHERE id = $2`, id, deal.Identifier.String()) + if err != nil { + return false, err + } + if n != 1 { + return false, fmt.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + if err != nil { + return false, xerrors.Errorf("adding deal to waiting pipeline: %w", err) + } + return true, nil + }) + + if err != nil { + log.Errorw("error inserting deal into DB", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + + if !comm { + log.Errorw("error committing deal into DB", "deal", deal) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + + return nil +} + +func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { + if !lo.Contains(m.miners, deal.Products.DDOV1.Provider) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Provider not available in Curio cluster", + }, nil + } + + if deal.Data.Size > abi.PaddedPieceSize(m.sm[deal.Products.DDOV1.Provider]) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Deal size is larger than the miner's sector size", + }, nil + } + + if deal.Products.DDOV1.AllocationId != nil { + if deal.Data.Size < abi.PaddedPieceSize(verifreg.MinimumVerifiedAllocationSize) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Verified piece size must be at least 1MB", + }, nil + } + + alloc, err := m.api.StateGetAllocation(ctx, deal.Products.DDOV1.Client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) + if err != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + }, xerrors.Errorf("getting allocation: %w", err) + } + + if alloc == nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Verified piece must have a valid allocation ID", + }, nil + } + + clientID, err := address.IDFromAddress(deal.Products.DDOV1.Client) + if err != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Invalid client address", + }, nil + } + + if alloc.Client != abi.ActorID(clientID) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "client address does not match the allocation client address", + }, nil + } + + prov, err := address.NewIDAddress(uint64(alloc.Provider)) + if err != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + }, xerrors.Errorf("getting provider address: %w", err) + } + + if !lo.Contains(m.miners, prov) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Allocation provider does not belong to the list of miners in Curio cluster", + }, nil + } + + if !deal.Data.PieceCID.Equals(alloc.Data) { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Allocation data CID does not match the piece CID", + }, nil + } + + if deal.Data.Size != alloc.Size { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Allocation size does not match the piece size", + }, nil + } + + if alloc.TermMin > miner.MaxSectorExpirationExtension-policy.SealRandomnessLookback { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Allocation term min is greater than the maximum sector expiration extension", + }, nil + } + } + + return nil, nil +} + +func (m *MK20) DealStatus(ctx context.Context, statusRequest *DealStatusRequest) (*DealStatusResponse, error) { + // TODO: implement this + return nil, nil +} + +// To be used later for when data source is minerID +//func validateMinerAddresses(madrs []abi.Multiaddrs, pcid cid.Cid, psize abi.PaddedPieceSize, rawSize int64) bool { +// var surls []*url.URL +// for _, adr := range madrs { +// surl, err := maurl.ToURL(multiaddr.Cast(adr)) +// if err != nil { +// continue +// } +// surls = append(surls, surl) +// } +// +// var validUrls []*url.URL +// +// for _, surl := range surls { +// if surl.Scheme == "ws" { +// surl.Scheme = "http" +// } +// +// if surl.Scheme == "wss" { +// surl.Scheme = "https" +// } +// +// if surl.Port() == "443" { +// surl.Host = surl.Hostname() +// } +// +// if surl.Port() == "80" { +// surl.Host = surl.Hostname() +// } +// +// resp, err := http.Head(surl.String() + "/piece/" + pcid.String()) +// if err != nil { +// continue +// } +// if resp.StatusCode != 200 { +// continue +// } +// +// if resp.Header.Get("Content-Length") != fmt.Sprint(psize) { +// continue +// } +// +// validUrls = append(validUrls, surl) +// } +// return len(validUrls) > 0 +//} diff --git a/market/mk20/types.go b/market/mk20/types.go new file mode 100644 index 000000000..5871dc02e --- /dev/null +++ b/market/mk20/types.go @@ -0,0 +1,154 @@ +package mk20 + +import ( + "net/http" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + + "github.com/filecoin-project/go-state-types/abi" +) + +// Deal represents a structure defining the details and components of a specific deal in the system. +type Deal struct { + + // Identifier represents a unique identifier for the deal in UUID format. + Identifier ulid.ULID `json:"identifier"` + + // Data represents the source of piece data and associated metadata. + Data DataSource `json:"data"` + + // Products represents a collection of product-specific information associated with a deal + Products Products `json:"products"` +} + +type Products struct { + DDOV1 *DDOV1 `json:"ddov1"` +} + +// DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. +type DataSource struct { + + // PieceCID represents the unique identifier for a piece of data, stored as a CID object. + PieceCID cid.Cid `json:"piececid"` + + // Size represents the size of the padded piece in the data source. + Size abi.PaddedPieceSize `json:"size"` + + // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. + Format PieceDataFormat `json:"format"` + + // SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. + SourceHTTP *DataSourceHTTP `json:"sourcehttp"` + + // SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. + SourceAggregate *DataSourceAggregate `json:"sourceaggregate"` + + // SourceOffline defines the data source for offline pieces, including raw size information. + SourceOffline *DataSourceOffline `json:"sourceoffline"` + // SourceHTTPPush // allow clients to push piece data after deal accepted, sort of like offline import + // SourceStorageProvider -> sp IDs/ipni, pieceCids +} + +// PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. +type PieceDataFormat struct { + + // Car represents the optional CAR file format, including its metadata and versioning details. + Car *FormatCar `json:"car"` + + // Aggregate holds a reference to the aggregated format of piece data. + Aggregate *FormatAggregate `json:"aggregate"` + + // Raw represents the raw format of the piece data, encapsulated as bytes. + Raw *FormatBytes `json:"raw"` +} + +// FormatCar represents the CAR (Content Addressable aRchive) format with version metadata for piece data serialization. +type FormatCar struct { + Version uint64 `json:"version"` +} + +// FormatAggregate represents the aggregated format for piece data, identified by its type. +type FormatAggregate struct { + + // Type specifies the type of aggregation for data pieces, represented by an AggregateType value. + Type AggregateType `json:"type"` + + // Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. + // The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate + Sub []PieceDataFormat `json:"sub"` +} + +// FormatBytes defines the raw byte representation of data as a format. +type FormatBytes struct{} + +// DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. +type DataSourceOffline struct { + RawSize uint64 `json:"rawsize"` +} + +// DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. +type DataSourceAggregate struct { + Pieces []DataSource `json:"pieces"` +} + +// DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. +type DataSourceHTTP struct { + + // RawSize specifies the raw size of the data in bytes. + RawSize uint64 `json:"rawsize"` + + // URLs lists the HTTP endpoints where the piece data can be fetched. + URLs []HttpUrl `json:"urls"` +} + +// HttpUrl represents an HTTP endpoint configuration for fetching piece data. +type HttpUrl struct { + + // URL specifies the HTTP endpoint where the piece data can be fetched. + URL string `json:"url"` + + // HTTPHeaders represents the HTTP headers associated with the URL. + HTTPHeaders http.Header `json:"httpheaders"` + + // Priority indicates the order preference for using the URL in requests, with lower values having higher priority. + Priority uint64 `json:"priority"` + + // Fallback indicates whether this URL serves as a fallback option when other URLs fail. + Fallback bool `json:"fallback"` +} + +// AggregateType represents an unsigned integer used to define the type of aggregation for data pieces in the system. +type AggregateType uint64 + +const ( + AggregateTypeNone AggregateType = iota + AggregateTypeV1 +) + +type ErrCode int + +const ( + Ok = 200 + ErrBadProposal = 400 + ErrMalformedDataSource = 400 + ErrUnsupportedDataSource = 422 + ErrUnsupportedProduct = 422 + ErrProductNotEnabled = 403 + ErrProductValidationRejected = 409 + ErrDealRejectedByMarket = 422 + ErrServiceMaintenance = 503 + ErrServiceOverloaded = 429 + ErrMarketNotEnabled = 440 + ErrDurationTooShort = 441 +) + +// TODO: Deal Status - HTTP +// TODO: Supported contractS - HTTP +// TODO: Client facing UI Page for SP +// TODO: Contract SP details pathway - sptool? +// TODO: Error codes use +// TODO: /PUT endpoint +// TODO: SPID data source +// TODO: Test contract +// TODO: ACLv1? diff --git a/market/mk20/utils.go b/market/mk20/utils.go new file mode 100644 index 000000000..252e78390 --- /dev/null +++ b/market/mk20/utils.go @@ -0,0 +1,450 @@ +package mk20 + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "math/bits" + "net/url" + "time" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-data-segment/datasegment" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +func (d *Deal) Validate() (bool, error) { + if d.Products.DDOV1 == nil { + return false, xerrors.Errorf("no products") + } + + return d.Data.Validate() +} + +func (d *DataSource) Validate() (bool, error) { + + if !d.PieceCID.Defined() { + return false, xerrors.Errorf("piece cid is not defined") + } + + if d.Size == 0 { + return false, xerrors.Errorf("piece size is 0") + } + + if d.SourceOffline != nil && d.SourceHTTP != nil && d.SourceAggregate != nil { + return false, xerrors.Errorf("multiple sources defined for data source") + } + + if d.SourceOffline == nil && d.SourceHTTP == nil && d.SourceAggregate == nil { + return false, xerrors.Errorf("no source defined for data source") + } + + var fcar, fagg, fraw bool + + if d.Format.Car != nil { + fcar = true + if d.Format.Car.Version != 1 && d.Format.Car.Version != 2 { + return false, xerrors.Errorf("car version not supported") + } + } + + if d.Format.Aggregate != nil { + fagg = true + + if d.Format.Aggregate.Type != AggregateTypeV1 { + return false, xerrors.Errorf("aggregate type not supported") + } + + if d.SourceAggregate != nil { + if len(d.SourceAggregate.Pieces) == 0 { + return false, xerrors.Errorf("no pieces in aggregate") + } + + for _, p := range d.SourceAggregate.Pieces { + if !p.PieceCID.Defined() { + return false, xerrors.Errorf("piece cid is not defined") + } + + if p.Size == 0 { + return false, xerrors.Errorf("piece size is 0") + } + + var ifcar, ifraw bool + + if p.Format.Car != nil { + ifcar = true + if p.Format.Car.Version != 1 && p.Format.Car.Version != 2 { + return false, xerrors.Errorf("car version not supported") + } + } + + if p.Format.Aggregate != nil { + return false, xerrors.Errorf("aggregate of aggregate is not supported") + } + + if p.Format.Raw != nil { + ifraw = true + } + + if !ifcar && !ifraw { + return false, xerrors.Errorf("no format defined for sub piece in aggregate") + } + + if ifcar && ifraw { + return false, xerrors.Errorf("multiple formats defined for sub piece in aggregate") + } + + if p.SourceAggregate != nil { + return false, xerrors.Errorf("aggregate of aggregate is not supported") + } + + if p.SourceOffline == nil && p.SourceHTTP == nil { + return false, xerrors.Errorf("no source defined for sub piece in aggregate") + } + + if p.SourceOffline != nil && p.SourceHTTP != nil { + return false, xerrors.Errorf("multiple sources defined for sub piece in aggregate") + } + + if p.SourceHTTP != nil { + if p.SourceHTTP.RawSize == 0 { + return false, xerrors.Errorf("raw size is 0 for sub piece in aggregate") + } + + if len(p.SourceHTTP.URLs) == 0 { + return false, xerrors.Errorf("no urls defined for sub piece in aggregate") + } + + for _, u := range d.SourceHTTP.URLs { + _, err := url.Parse(u.URL) + if err != nil { + return false, xerrors.Errorf("invalid url") + } + } + } + + if p.SourceOffline != nil { + if p.SourceOffline.RawSize == 0 { + return false, xerrors.Errorf("raw size is 0 for sub piece in aggregate") + } + } + + } + } + } + + if d.Format.Raw != nil { + fraw = true + } + + if !fcar && !fagg && !fraw { + return false, xerrors.Errorf("no format defined") + } + + if fcar && fagg || fcar && fraw || fagg && fraw { + return false, xerrors.Errorf("multiple formats defined") + } + + if d.SourceHTTP != nil { + if d.SourceHTTP.RawSize == 0 { + return false, xerrors.Errorf("raw size is 0") + } + + if len(d.SourceHTTP.URLs) == 0 { + return false, xerrors.Errorf("no urls defined") + } + + for _, u := range d.SourceHTTP.URLs { + _, err := url.Parse(u.URL) + if err != nil { + return false, xerrors.Errorf("invalid url") + } + } + } + + if d.SourceOffline != nil { + if d.SourceOffline.RawSize == 0 { + return false, xerrors.Errorf("raw size is 0") + } + } + + raw, err := d.RawSize() + if err != nil { + return false, err + } + + if padreader.PaddedSize(raw).Padded() != d.Size { + return false, xerrors.Errorf("invalid size") + } + + return true, nil +} + +func (d *DataSource) RawSize() (uint64, error) { + if d.Format.Aggregate != nil { + if d.Format.Aggregate.Type == AggregateTypeV1 { + if d.SourceAggregate != nil { + var pinfos []abi.PieceInfo + for _, piece := range d.SourceAggregate.Pieces { + pinfos = append(pinfos, abi.PieceInfo{ + PieceCID: piece.PieceCID, + Size: piece.Size, + }) + } + _, asize, err := datasegment.ComputeDealPlacement(pinfos) + if err != nil { + return 0, err + } + next := 1 << (64 - bits.LeadingZeros64(asize+256)) + if abi.PaddedPieceSize(next) != d.Size { + return 0, xerrors.Errorf("invalid aggregate size") + } + + a, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), pinfos) + if err != nil { + return 0, err + } + + return uint64(a.DealSize.Unpadded()), nil + } + } + } + + if d.SourceHTTP != nil { + return d.SourceHTTP.RawSize, nil + } + + if d.SourceOffline != nil { + return d.SourceOffline.RawSize, nil + } + return 0, xerrors.Errorf("no source defined") +} + +type DBDeal struct { + Identifier string `db:"id"` + PieceCID string `db:"piece_cid"` + Size int64 `db:"size"` + Format json.RawMessage `db:"format"` + SourceHTTP json.RawMessage `db:"source_http"` + SourceAggregate json.RawMessage `db:"source_aggregate"` + SourceOffline json.RawMessage `db:"source_offline"` + DDOv1 json.RawMessage `db:"ddov1"` +} + +func (d *Deal) ToDBDeal() (*DBDeal, error) { + + // Marshal Format (always present) + formatBytes, err := json.Marshal(d.Data.Format) + if err != nil { + return nil, fmt.Errorf("marshal format: %w", err) + } + + // Marshal SourceHTTP (optional) + var sourceHTTPBytes []byte + if d.Data.SourceHTTP != nil { + sourceHTTPBytes, err = json.Marshal(d.Data.SourceHTTP) + if err != nil { + return nil, fmt.Errorf("marshal source_http: %w", err) + } + } else { + sourceHTTPBytes = []byte("null") + } + + // Marshal SourceAggregate (optional) + var sourceAggregateBytes []byte + if d.Data.SourceAggregate != nil { + sourceAggregateBytes, err = json.Marshal(d.Data.SourceAggregate) + if err != nil { + return nil, fmt.Errorf("marshal source_aggregate: %w", err) + } + } else { + sourceAggregateBytes = []byte("null") + } + + // Marshal SourceOffline (optional) + var sourceOfflineBytes []byte + if d.Data.SourceOffline != nil { + sourceOfflineBytes, err = json.Marshal(d.Data.SourceOffline) + if err != nil { + return nil, fmt.Errorf("marshal source_offline: %w", err) + } + } else { + sourceOfflineBytes = []byte("null") + } + + var ddov1 []byte + if d.Products.DDOV1 != nil { + ddov1, err = json.Marshal(d.Products.DDOV1) + if err != nil { + return nil, fmt.Errorf("marshal ddov1: %w", err) + } + } else { + ddov1 = []byte("null") + } + + return &DBDeal{ + Identifier: d.Identifier.String(), + PieceCID: d.Data.PieceCID.String(), + Size: int64(d.Data.Size), + Format: formatBytes, + SourceHTTP: sourceHTTPBytes, + SourceAggregate: sourceAggregateBytes, + SourceOffline: sourceOfflineBytes, + DDOv1: ddov1, + }, nil +} + +func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { + dbDeal, err := d.ToDBDeal() + if err != nil { + return xerrors.Errorf("to db deal: %w", err) + } + + n, err := tx.Exec(`INSERT INTO deals (id, piece_cid, size, format, source_http, source_aggregate, source_offline, ddov1) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + dbDeal.Identifier, + dbDeal.PieceCID, + dbDeal.Size, + dbDeal.Format, + dbDeal.SourceHTTP, + dbDeal.SourceAggregate, + dbDeal.SourceOffline, + dbDeal.DDOv1) + if err != nil { + return xerrors.Errorf("insert deal: %w", err) + } + if n != 1 { + return xerrors.Errorf("insert deal: expected 1 row affected, got %d", n) + } + return nil +} + +func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { + var dbDeal []DBDeal + err := tx.Select(&dbDeal, `SELECT * FROM deals WHERE id = $1`, id.String()) + if err != nil { + return nil, xerrors.Errorf("getting deal from DB: %w", err) + } + if len(dbDeal) != 1 { + return nil, xerrors.Errorf("expected 1 deal, got %d", len(dbDeal)) + } + return dbDeal[0].ToDeal() +} + +func DealFromDB(ctx context.Context, db *harmonydb.DB, id ulid.ULID) (*Deal, error) { + var dbDeal []DBDeal + err := db.Select(ctx, &dbDeal, `SELECT * FROM deals WHERE id = $1`, id.String()) + if err != nil { + return nil, xerrors.Errorf("getting deal from DB: %w", err) + } + if len(dbDeal) != 1 { + return nil, xerrors.Errorf("expected 1 deal, got %d", len(dbDeal)) + } + return dbDeal[0].ToDeal() +} + +func (d *DBDeal) ToDeal() (*Deal, error) { + var ds DataSource + var products Products + + // Unmarshal each field into the corresponding sub-structs (nil will remain nil if json is "null" or empty) + if err := json.Unmarshal(d.Format, &ds.Format); err != nil { + return nil, fmt.Errorf("unmarshal format: %w", err) + } + + if len(d.SourceHTTP) > 0 && string(d.SourceHTTP) != "null" { + var sh DataSourceHTTP + if err := json.Unmarshal(d.SourceHTTP, &sh); err != nil { + return nil, fmt.Errorf("unmarshal source_http: %w", err) + } + ds.SourceHTTP = &sh + } + + if len(d.SourceAggregate) > 0 && string(d.SourceAggregate) != "null" { + var sa DataSourceAggregate + if err := json.Unmarshal(d.SourceAggregate, &sa); err != nil { + return nil, fmt.Errorf("unmarshal source_aggregate: %w", err) + } + ds.SourceAggregate = &sa + } + + if len(d.SourceOffline) > 0 && string(d.SourceOffline) != "null" { + var so DataSourceOffline + if err := json.Unmarshal(d.SourceOffline, &so); err != nil { + return nil, fmt.Errorf("unmarshal source_offline: %w", err) + } + ds.SourceOffline = &so + } + + if len(d.DDOv1) > 0 && string(d.DDOv1) != "null" { + if err := json.Unmarshal(d.DDOv1, &products.DDOV1); err != nil { + return nil, fmt.Errorf("unmarshal ddov1: %w", err) + } + } + + // Convert identifier + id, err := ulid.Parse(d.Identifier) + if err != nil { + return nil, fmt.Errorf("parse identifier: %w", err) + } + + // Convert CID + c, err := cid.Decode(d.PieceCID) + if err != nil { + return nil, fmt.Errorf("decode piece_cid: %w", err) + } + + // Assign remaining fields + ds.PieceCID = c + ds.Size = abi.PaddedPieceSize(d.Size) + + return &Deal{ + Identifier: id, + Data: ds, + Products: products, + }, nil +} + +func DBDealsToDeals(deals []*DBDeal) ([]*Deal, error) { + var result []*Deal + for _, d := range deals { + deal, err := d.ToDeal() + if err != nil { + return nil, err + } + result = append(result, deal) + } + return result, nil +} + +type ProviderDealRejectionInfo struct { + HTTPCode int + Reason string +} + +type DealStatusRequest struct { + Identifier string `json:"identifier"` + IdentifierType uint64 `json:"identifiertype"` + ContractAddress string `json:"contractaddress"` +} + +type DealStatusResponse struct { + Complete bool `json:"complete"` + Error bool `json:"error"` + ErrorMsg string `json:"errormsg"` +} + +type SupportedContracts struct { + Contracts []string `json:"contracts"` +} + +func NewULID() (ulid.ULID, error) { + return ulid.New(ulid.Timestamp(time.Now()), rand.Reader) +} diff --git a/market/retrieval/piecehandler.go b/market/retrieval/piecehandler.go index 94bde5119..931b0eb5b 100644 --- a/market/retrieval/piecehandler.go +++ b/market/retrieval/piecehandler.go @@ -12,8 +12,6 @@ import ( "github.com/ipfs/go-cid" "go.opencensus.io/stats" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/curio/lib/cachedreader" "github.com/filecoin-project/curio/market/retrieval/remoteblockstore" ) @@ -73,15 +71,16 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { return } - setHeaders(w, pieceCid, contentType) - serveContent(w, r, size, reader) + setHeaders(w, pieceCid, contentType, int64(size)) + serveContent(w, r, reader) stats.Record(ctx, remoteblockstore.HttpPieceByCid200ResponseCount.M(1)) stats.Record(ctx, remoteblockstore.HttpPieceByCidRequestDuration.M(float64(time.Since(startTime).Milliseconds()))) } -func setHeaders(w http.ResponseWriter, pieceCid cid.Cid, contentType string) { +func setHeaders(w http.ResponseWriter, pieceCid cid.Cid, contentType string, size int64) { w.Header().Set("Vary", "Accept-Encoding") + w.Header().Set("Content-Length", fmt.Sprintf("%d", size)) w.Header().Set("Cache-Control", "public, max-age=29030400, immutable") w.Header().Set("Content-Type", contentType) if contentType != "application/octet-stream" { @@ -98,7 +97,7 @@ func setHeaders(w http.ResponseWriter, pieceCid cid.Cid, contentType string) { } -func serveContent(res http.ResponseWriter, req *http.Request, size abi.UnpaddedPieceSize, content io.ReadSeeker) { +func serveContent(res http.ResponseWriter, req *http.Request, content io.ReadSeeker) { // Note that the last modified time is a constant value because the data // in a piece identified by a cid will never change. @@ -109,6 +108,5 @@ func serveContent(res http.ResponseWriter, req *http.Request, size abi.UnpaddedP } // Send the content - res.Header().Set("Content-Length", fmt.Sprintf("%d", size)) http.ServeContent(res, req, "", lastModified, content) } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 2edfcc957..f3d29741b 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -69,6 +69,7 @@ type itask struct { Announce bool `db:"announce"` ChainDealId abi.DealID `db:"chain_deal_id"` IsDDO bool `db:"is_ddo"` + Mk20 bool `db:"mk20"` } func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { @@ -78,29 +79,51 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do ctx := context.Background() err = i.db.Select(ctx, &tasks, `SELECT - p.uuid, - p.sp_id, - p.sector, - p.piece_cid, - p.piece_size, - p.sector_offset, - p.reg_seal_proof, - p.raw_size, - p.should_index, - p.announce, - p.is_ddo, - COALESCE(d.chain_deal_id, 0) AS chain_deal_id -- If NULL, return 0 + p.uuid, + p.sp_id, + p.sector, + p.piece_cid, + p.piece_size, + p.sector_offset, + p.reg_seal_proof, + p.raw_size, + p.should_index, + p.announce, + p.is_ddo, + COALESCE(d.chain_deal_id, 0) AS chain_deal_id, + false AS mk20 FROM - market_mk12_deal_pipeline p + market_mk12_deal_pipeline p LEFT JOIN - market_mk12_deals d - ON p.uuid = d.uuid AND p.sp_id = d.sp_id + market_mk12_deals d + ON p.uuid = d.uuid AND p.sp_id = d.sp_id LEFT JOIN - market_direct_deals md - ON p.uuid = md.uuid AND p.sp_id = md.sp_id + market_direct_deals md + ON p.uuid = md.uuid AND p.sp_id = md.sp_id WHERE - p.indexing_task_id = $1; - ;`, taskID) + p.indexing_task_id = $1 + + UNION ALL + + SELECT + id AS uuid, + sp_id, + sector, + piece_cid, + piece_size, + sector_offset, + reg_seal_proof, + raw_size, + indexing as should_index, + announce, + TRUE AS is_ddo, + 0 AS chain_deal_id, + TRUE AS mk20 + FROM + market_mk20_pipeline p + WHERE + p.indexing_task_id = $1; + `, taskID) if err != nil { return false, xerrors.Errorf("getting indexing params: %w", err) } @@ -124,7 +147,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do if err != nil { return false, err } - log.Infow("Piece already indexed or should not be indexed", "piece_cid", task.PieceCid, "indexed", indexed, "should_index", task.ShouldIndex, "uuid", task.UUID, "sp_id", task.SpID, "sector", task.Sector) + log.Infow("Piece already indexed or should not be indexed", "piece_cid", task.PieceCid, "indexed", indexed, "should_index", task.ShouldIndex, "id", task.UUID, "sp_id", task.SpID, "sector", task.Sector) return true, nil } @@ -216,7 +239,7 @@ loop: } blocksPerSecond := float64(blocks) / time.Since(start).Seconds() - log.Infow("Piece indexed", "piece_cid", task.PieceCid, "uuid", task.UUID, "sp_id", task.SpID, "sector", task.Sector, "blocks", blocks, "blocks_per_second", blocksPerSecond) + log.Infow("Piece indexed", "piece_cid", task.PieceCid, "id", task.UUID, "sp_id", task.SpID, "sector", task.Sector, "blocks", blocks, "blocks_per_second", blocksPerSecond) return true, nil } @@ -232,22 +255,44 @@ func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID // If IPNI is disabled then mark deal as complete otherwise just mark as indexed if i.cfg.Market.StorageMarketConfig.IPNI.Disable { - n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, indexing_task_id = NULL, + if task.Mk20 { + n, err := i.db.Exec(ctx, `UPDATE market_mk20_pipeline SET indexed = TRUE, indexing_task_id = NULL, + complete = TRUE WHERE id = $1 AND indexing_task_id = $2`, task.UUID, taskID) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } + } else { + n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, indexing_task_id = NULL, complete = TRUE WHERE uuid = $1 AND indexing_task_id = $2`, task.UUID, taskID) - if err != nil { - return xerrors.Errorf("store indexing success: updating pipeline: %w", err) - } - if n != 1 { - return xerrors.Errorf("store indexing success: updated %d rows", n) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } } } else { - n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, indexing_task_id = NULL + if task.Mk20 { + n, err := i.db.Exec(ctx, `UPDATE market_mk20_pipeline SET indexed = TRUE, indexing_task_id = NULL + WHERE id = $1 AND indexing_task_id = $2`, task.UUID, taskID) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } + } else { + n, err := i.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET indexed = TRUE, indexing_task_id = NULL WHERE uuid = $1 AND indexing_task_id = $2`, task.UUID, taskID) - if err != nil { - return xerrors.Errorf("store indexing success: updating pipeline: %w", err) - } - if n != 1 { - return xerrors.Errorf("store indexing success: updated %d rows", n) + if err != nil { + return xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("store indexing success: updated %d rows", n) + } } } @@ -265,10 +310,20 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T // Accept any task which should not be indexed as // it does not require storage access var id int64 - err := i.db.QueryRow(ctx, `SELECT indexing_task_id - FROM market_mk12_deal_pipeline - WHERE should_index = FALSE AND - indexing_task_id = ANY ($1) ORDER BY indexing_task_id LIMIT 1`, indIDs).Scan(&id) + err := i.db.QueryRow(ctx, `SELECT indexing_task_id + FROM market_mk12_deal_pipeline + WHERE should_index = FALSE + AND indexing_task_id = ANY ($1) + + UNION ALL + + SELECT indexing_task_id + FROM market_mk20_pipeline + WHERE indexing = FALSE + AND indexing_task_id = ANY ($1) + + ORDER BY indexing_task_id + LIMIT 1;`, indIDs).Scan(&id) if err == nil { ret := harmonytask.TaskID(id) return &ret, nil @@ -287,11 +342,17 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T panic("storiface.FTUnsealed != 1") } - err = i.db.Select(ctx, &tasks, ` - SELECT dp.indexing_task_id, dp.sp_id, dp.sector, l.storage_id FROM market_mk12_deal_pipeline dp - INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num - WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1 -`, indIDs) + err = i.db.Select(ctx, &tasks, `SELECT dp.indexing_task_id, dp.sp_id, dp.sector, l.storage_id + FROM market_mk12_deal_pipeline dp + INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num + WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1 + + UNION ALL + + SELECT dp.indexing_task_id, dp.sp_id, dp.sector, l.storage_id + FROM market_mk20_pipeline dp + INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num + WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1`, indIDs) if err != nil { return nil, xerrors.Errorf("getting tasks: %w", err) } @@ -340,35 +401,60 @@ func (i *IndexingTask) schedule(ctx context.Context, taskFunc harmonytask.AddTas taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule - var pendings []struct { + var mk12Pendings []struct { UUID string `db:"uuid"` } // Indexing job must be created for every deal to make sure piece details are inserted in DB // even if we don't want to index it. If piece is not supposed to be indexed then it will handled // by the Do() - err := i.db.Select(ctx, &pendings, `SELECT uuid FROM market_mk12_deal_pipeline + err := tx.Select(&mk12Pendings, `SELECT uuid FROM market_mk12_deal_pipeline WHERE sealed = TRUE AND indexing_task_id IS NULL AND indexed = FALSE ORDER BY indexing_created_at ASC LIMIT 1;`) if err != nil { - return false, xerrors.Errorf("getting pending indexing tasks: %w", err) + return false, xerrors.Errorf("getting pending mk12 indexing tasks: %w", err) } - if len(pendings) == 0 { - return false, nil + if len(mk12Pendings) > 0 { + pending := mk12Pendings[0] + + _, err = tx.Exec(`UPDATE market_mk12_deal_pipeline SET indexing_task_id = $1 + WHERE indexing_task_id IS NULL AND uuid = $2`, id, pending.UUID) + if err != nil { + return false, xerrors.Errorf("updating mk12 indexing task id: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil } - pending := pendings[0] + var mk20Pendings []struct { + UUID string `db:"id"` + } - _, err = tx.Exec(`UPDATE market_mk12_deal_pipeline SET indexing_task_id = $1 - WHERE indexing_task_id IS NULL AND uuid = $2`, id, pending.UUID) + err = tx.Select(&mk20Pendings, `SELECT id FROM market_mk20_pipeline + WHERE sealed = TRUE + AND indexing_task_id IS NULL + AND indexed = FALSE + ORDER BY indexing_created_at ASC LIMIT 1;`) + if err != nil { + return false, xerrors.Errorf("getting mk20 pending indexing tasks: %w", err) + } + + if len(mk20Pendings) == 0 { + return false, nil + } + + pending := mk20Pendings[0] + _, err = tx.Exec(`UPDATE market_mk20_pipeline SET indexing_task_id = $1 + WHERE indexing_task_id IS NULL AND id = $2`, id, pending.UUID) if err != nil { - return false, xerrors.Errorf("updating indexing task id: %w", err) + return false, xerrors.Errorf("updating mk20 indexing task id: %w", err) } - stop = false // we found a task to schedule, keep going + stop = false return true, nil }) } diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 622263e7f..4c0eae87b 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -348,6 +348,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun var stop bool for !stop { var markComplete *string + var mk20 bool taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule @@ -355,20 +356,41 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun var pendings []itask err := tx.Select(&pendings, `SELECT - uuid, - sp_id, - sector, - piece_cid, - piece_size, - sector_offset, - reg_seal_proof, - raw_size, - should_index, - announce - FROM market_mk12_deal_pipeline + uuid, + sp_id, + sector, + piece_cid, + piece_size, + sector_offset, + reg_seal_proof, + raw_size, + should_index, + announce, + FALSE as mk20 + FROM market_mk12_deal_pipeline WHERE sealed = TRUE - AND indexed = TRUE - AND complete = FALSE + AND indexed = TRUE + AND complete = FALSE + + UNION ALL + + SELECT + id AS uuid, + sp_id, + sector, + piece_cid, + piece_size, + sector_offset, + reg_seal_proof, + raw_size, + indexing AS should_index, + announce, + TRUE as mk20 + FROM market_mk20_pipeline + WHERE sealed = TRUE + AND indexed = TRUE + AND complete = FALSE + ORDER BY indexing_created_at ASC LIMIT 1;`) if err != nil { @@ -384,9 +406,14 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun // Skip IPNI if deal says not to announce or not to index (fast retrievals). If we announce without // indexing, it will cause issue with retrievals. if !p.Announce || !p.ShouldIndex { - n, err := tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID) + var n int + if p.Mk20 { + n, err = tx.Exec(`UPDATE market_mk20_pipeline SET complete = TRUE WHERE id = $1`, p.UUID) + } else { + n, err = tx.Exec(`UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1`, p.UUID) + } if err != nil { - return false, xerrors.Errorf("store IPNI success: updating pipeline (1): %w", err) + return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) } if n != 1 { return false, xerrors.Errorf("store IPNI success: updated %d rows", n) @@ -461,6 +488,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun ilog.Infof("Another IPNI announce task already present for piece %s in deal %s", p.PieceCid, p.UUID) // SET "complete" status to true for this deal, so it is not considered next time markComplete = &p.UUID + mk20 = p.Mk20 stop = false // we found a sector to work on, keep going return true, nil } @@ -468,6 +496,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.UUID) // SET "complete" status to true for this deal, so it is not considered next time markComplete = &p.UUID + mk20 = p.Mk20 stop = false // we found a sector to work on, keep going return false, nil } @@ -479,7 +508,13 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun }) if markComplete != nil { - n, err := I.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1 AND complete = FALSE`, *markComplete) + var n int + var err error + if mk20 { + n, err = I.db.Exec(ctx, `UPDATE market_mk20_pipeline SET complete = TRUE WHERE id = $1 AND complete = FALSE`, *markComplete) + } else { + n, err = I.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET complete = TRUE WHERE uuid = $1 AND complete = FALSE`, *markComplete) + } if err != nil { log.Errorf("store IPNI success: updating pipeline (2): %s", err) } diff --git a/tasks/seal/poller_commit_msg.go b/tasks/seal/poller_commit_msg.go index 312162248..68ddc65ae 100644 --- a/tasks/seal/poller_commit_msg.go +++ b/tasks/seal/poller_commit_msg.go @@ -161,7 +161,14 @@ func (s *SealPoller) pollCommitMsgLanded(ctx context.Context, task pollTask) err if err != nil { return false, xerrors.Errorf("update market_mk12_deal_pipeline: %w", err) } - log.Debugw("marked deals as sealed", "sp", task.SpID, "sector", task.SectorNumber, "count", n) + log.Debugw("marked mk12 deals as sealed", "sp", task.SpID, "sector", task.SectorNumber, "count", n) + + n, err = tx.Exec(`UPDATE market_mk20_pipeline SET sealed = TRUE WHERE sp_id = $1 AND sector = $2 AND sealed = FALSE`, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update market_mk20_pipeline: %w", err) + } + log.Debugw("marked mk20 deals as sealed", "sp", task.SpID, "sector", task.SectorNumber, "count", n) + return true, nil } } diff --git a/tasks/seal/task_movestorage.go b/tasks/seal/task_movestorage.go index 13017db75..3414b217b 100644 --- a/tasks/seal/task_movestorage.go +++ b/tasks/seal/task_movestorage.go @@ -67,8 +67,16 @@ func (m *MoveStorageTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) } _, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - // Create a indexing task - _, err = tx.Exec(`SELECT create_indexing_task($1, $2)`, taskID, "sectors_sdr_pipeline") + // Set indexing_created_at to Now() to allow new indexing tasks + _, err = tx.Exec(` + UPDATE market_mk20_pipeline + SET indexing_created_at = NOW() + WHERE sp_id = $1 AND sector = $2; + + UPDATE market_mk12_deal_pipeline + SET indexing_created_at = NOW() + WHERE sp_id = $1 AND sector = $2; + `, task.SpID, task.SectorNumber) if err != nil { return false, fmt.Errorf("error creating indexing task: %w", err) } diff --git a/tasks/snap/task_movestorage.go b/tasks/snap/task_movestorage.go index 4680d08ad..39e6c10a5 100644 --- a/tasks/snap/task_movestorage.go +++ b/tasks/snap/task_movestorage.go @@ -71,8 +71,16 @@ func (m *MoveStorageTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) } _, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - // Create an indexing task - _, err = tx.Exec(`SELECT create_indexing_task($1, $2)`, taskID, "sectors_snap_pipeline") + // Set indexing_created_at to Now() to allow new indexing tasks + _, err = tx.Exec(` + UPDATE market_mk20_pipeline + SET indexing_created_at = NOW() + WHERE sp_id = $1 AND sector = $2; + + UPDATE market_mk12_deal_pipeline + SET indexing_created_at = NOW() + WHERE sp_id = $1 AND sector = $2; + `, task.SpID, task.SectorNumber) if err != nil { return false, fmt.Errorf("error creating indexing task: %w", err) } diff --git a/tasks/snap/task_submit.go b/tasks/snap/task_submit.go index 67e7c21e8..f7caa2f63 100644 --- a/tasks/snap/task_submit.go +++ b/tasks/snap/task_submit.go @@ -804,7 +804,13 @@ func (s *SubmitTask) updateLanded(ctx context.Context, tx *harmonydb.Tx, spId, s if err != nil { return xerrors.Errorf("update market_mk12_deal_pipeline: %w", err) } - log.Debugw("marked deals as sealed", "sp", spId, "sector", sectorNum, "count", n) + log.Debugw("marked mk12 deals as sealed", "sp", spId, "sector", sectorNum, "count", n) + + n, err = tx.Exec(`UPDATE market_mk20_pipeline SET sealed = TRUE WHERE sp_id = $1 AND sector = $2 AND sealed = FALSE`, spId, sectorNum) + if err != nil { + return xerrors.Errorf("update market_mk20_pipeline: %w", err) + } + log.Debugw("marked mk20 deals as sealed", "sp", spId, "sector", sectorNum, "count", n) } } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go new file mode 100644 index 000000000..55b0812e5 --- /dev/null +++ b/tasks/storage-market/mk20.go @@ -0,0 +1,1085 @@ +package storage_market + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "runtime" + "strconv" + "time" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin" + verifreg13 "github.com/filecoin-project/go-state-types/builtin/v13/verifreg" + "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/market/mk20" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/proofs" + "github.com/filecoin-project/lotus/chain/types" + lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece" +) + +type MK20PipelinePiece struct { + ID string `db:"id"` + SPID int64 `db:"sp_id"` + Client string `db:"client"` + Contract string `db:"contract"` + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + Offline bool `db:"offline"` + URL *string `db:"url"` // Nullable fields use pointers + Indexing bool `db:"indexing"` + Announce bool `db:"announce"` + AllocationID *int64 `db:"allocation_id"` // Nullable fields use pointers + Duration *int64 `db:"duration"` // Nullable fields use pointers + PieceAggregation int `db:"piece_aggregation"` + + Started bool `db:"started"` + + Downloaded bool `db:"downloaded"` + + CommTaskID *int64 `db:"commp_task_id"` + AfterCommp bool `db:"after_commp"` + + DealAggregation int `db:"deal_aggregation"` + AggregationIndex int64 `db:"aggr_index"` + AggregationTaskID *int64 `db:"agg_task_id"` + Aggregated bool `db:"aggregated"` + + Sector *int64 `db:"sector"` // Nullable fields use pointers + RegSealProof *int `db:"reg_seal_proof"` // Nullable fields use pointers + SectorOffset *int64 `db:"sector_offset"` // Nullable fields use pointers + + IndexingCreatedAt *time.Time `db:"indexing_created_at"` // Nullable fields use pointers + IndexingTaskID *int64 `db:"indexing_task_id"` + Indexed bool `db:"indexed"` +} + +func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { + go d.pipelineInsertLoop(ctx) + // Catch any panics if encountered as we are working with user provided data + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + + log.Errorf("panic occurred: %v\n%s", r, trace[:n]) + } + }() + d.processMK20DealPieces(ctx) + d.processMK20DealAggregation(ctx) + d.processMK20DealIngestion(ctx) +} + +func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.insertDDODealInPipeline(ctx) + } + } +} + +func (d *CurioStorageDealMarket) insertDDODealInPipeline(ctx context.Context) { + var deals []string + rows, err := d.db.Query(ctx, `SELECT id from market_mk20_pipeline_waiting`) + if err != nil { + log.Errorf("querying mk20 pipeline waiting: %s", err) + return + } + for rows.Next() { + var dealID string + err = rows.Scan(&dealID) + if err != nil { + log.Errorf("scanning mk20 pipeline waiting: %s", err) + return + } + deals = append(deals, dealID) + } + + if err := rows.Err(); err != nil { + log.Errorf("iterating over mk20 pipeline waiting: %s", err) + return + } + var dealIDs []ulid.ULID + for _, dealID := range deals { + id, err := ulid.Parse(dealID) + if err != nil { + log.Errorf("parsing deal id: %s", err) + return + } + dealIDs = append(dealIDs, id) + } + if len(dealIDs) == 0 { + return + } + for _, id := range dealIDs { + comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + deal, err := mk20.DealFromTX(tx, id) + if err != nil { + return false, xerrors.Errorf("getting deal from db: %w", err) + } + err = insertPiecesInTransaction(ctx, tx, deal) + if err != nil { + return false, xerrors.Errorf("inserting pieces in db: %w", err) + } + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) + } + return true, nil + }) + if err != nil { + log.Errorf("inserting deal in pipeline: %s", err) + continue + } + if !comm { + log.Errorf("inserting deal in pipeline: commit failed") + continue + } + } +} + +func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20.Deal) error { + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return fmt.Errorf("getting provider ID: %w", err) + } + + ddo := deal.Products.DDOV1 + data := deal.Data + dealID := deal.Identifier.String() + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + var aggregation interface{} + if data.Format.Aggregate != nil { + aggregation = data.Format.Aggregate.Type + } else { + aggregation = nil + } + + // Insert pipeline when Data source is HTTP + if data.SourceHTTP != nil { + var pieceID int64 + // Attempt to select the piece ID first + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, data.PieceCID.String(), data.Size).Scan(&pieceID) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // Piece does not exist, attempt to insert + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, TRUE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id`, data.PieceCID.String(), int64(data.Size), int64(data.SourceHTTP.RawSize)).Scan(&pieceID) + if err != nil { + return xerrors.Errorf("inserting new parked piece and getting id: %w", err) + } + } else { + // Some other error occurred during select + return xerrors.Errorf("checking existing parked piece: %w", err) + } + } + + var refIds []int64 + + // Add parked_piece_refs + for _, src := range data.SourceHTTP.URLs { + var refID int64 + + headers, err := json.Marshal(src.HTTPHeaders) + if err != nil { + return xerrors.Errorf("marshaling headers: %w", err) + } + + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + VALUES ($1, $2, $3, TRUE) RETURNING ref_id`, pieceID, src.URL, headers).Scan(&refID) + if err != nil { + return xerrors.Errorf("inserting parked piece ref: %w", err) + } + refIds = append(refIds, refID) + } + + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, + dealID, data.PieceCID.String(), data.Size, refIds) + if err != nil { + return xerrors.Errorf("inserting mk20 download pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting mk20 download pipeline: %d rows affected", n) + } + + n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, + piece_size, raw_size, offline, indexing, announce, + allocation_id, duration, piece_aggregation, started) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE)`, + dealID, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), + data.Size, int64(data.SourceHTTP.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, + allocationID, ddo.Duration, aggregation) + if err != nil { + return xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + return nil + } + + // INSERT Pipeline when data source is offline + if deal.Data.SourceOffline != nil { + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, + piece_size, raw_size, offline, indexing, announce, + allocation_id, duration, piece_aggregation) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, + dealID, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), + data.Size, int64(data.SourceHTTP.RawSize), true, ddo.Indexing, ddo.AnnounceToIPNI, + allocationID, ddo.Duration, aggregation) + if err != nil { + return xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + return nil + } + + // Insert pipeline when data source is aggregate + if deal.Data.SourceAggregate != nil { + + // Find all unique pieces where data source is HTTP + type downloadkey struct { + ID string + PieceCID cid.Cid + Size abi.PaddedPieceSize + } + toDownload := make(map[downloadkey][]mk20.HttpUrl) + existing := make(map[downloadkey]*int64) + offlinelist := make(map[downloadkey]struct{}) + + for _, piece := range deal.Data.SourceAggregate.Pieces { + if piece.SourceHTTP != nil { + urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] + if ok { + toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = append(urls, piece.SourceHTTP.URLs...) + } else { + toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = piece.SourceHTTP.URLs + existing[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = nil + } + } + if piece.SourceOffline != nil { + offlinelist[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = struct{}{} + } + } + + pqBatch := &pgx.Batch{} + pqBatchSize := 20000 + + for k, _ := range toDownload { + pqBatch.Queue(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, k.PieceCID.String(), int64(k.Size)).QueryRow(func(row pgx.Row) error { + var id int64 + err = row.Scan(&id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + return xerrors.Errorf("scanning parked piece id: %w", err) + } + existing[k] = &id + return nil + }) + if pqBatch.Len() > pqBatchSize { + res := tx.SendBatch(ctx, pqBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece query batch: %w", err) + } + pqBatch = &pgx.Batch{} + } + } + + if pqBatch.Len() > 0 { + res := tx.SendBatch(ctx, pqBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece query batch: %w", err) + } + } + + piBatch := &pgx.Batch{} + piBatchSize := 10000 + for k, v := range existing { + if v == nil { + piBatch.Queue(`INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, FALSE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id`, k.PieceCID.String(), int64(k.Size), int64(k.Size)).QueryRow(func(row pgx.Row) error { + var id int64 + err = row.Scan(&id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + return xerrors.Errorf("scanning parked piece id: %w", err) + } + v = &id + return nil + }) + if piBatch.Len() > piBatchSize { + res := tx.SendBatch(ctx, piBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece insert batch: %w", err) + } + piBatch = &pgx.Batch{} + } + } + } + + if piBatch.Len() > 0 { + res := tx.SendBatch(ctx, piBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece insert batch: %w", err) + } + } + + prBatch := &pgx.Batch{} + prBatchSize := 10000 + downloadMap := make(map[downloadkey][]int64) + + for k, v := range existing { + if v == nil { + return xerrors.Errorf("missing parked piece for %s", k.PieceCID.String()) + } + var refIds []int64 + urls := toDownload[downloadkey{PieceCID: k.PieceCID, Size: k.Size}] + for _, src := range urls { + headers, err := json.Marshal(src.HTTPHeaders) + if err != nil { + return xerrors.Errorf("marshal headers: %w", err) + } + prBatch.Queue(`INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) VALUES ($1, $2, $3, FALSE) RETURNING ref_id`, + *v, src.URL, headers).QueryRow(func(row pgx.Row) error { + var id int64 + err = row.Scan(&id) + if err != nil { + return xerrors.Errorf("scanning parked piece ref id: %w", err) + } + refIds = append(refIds, id) + return nil + }) + + if prBatch.Len() > prBatchSize { + res := tx.SendBatch(ctx, prBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece ref insert batch: %w", err) + } + prBatch = &pgx.Batch{} + } + } + downloadMap[downloadkey{ID: dealID, PieceCID: k.PieceCID, Size: k.Size}] = refIds + + } + + if prBatch.Len() > 0 { + res := tx.SendBatch(ctx, prBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece ref insert batch: %w", err) + } + } + + mdBatch := &pgx.Batch{} + mdBatchSize := 20000 + for k, v := range downloadMap { + mdBatch.Queue(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, + k.ID, k.PieceCID.String(), k.Size, v) + if mdBatch.Len() > mdBatchSize { + res := tx.SendBatch(ctx, mdBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + } + mdBatch = &pgx.Batch{} + } + } + if mdBatch.Len() > 0 { + res := tx.SendBatch(ctx, mdBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + } + } + + pBatch := &pgx.Batch{} + pBatchSize := 4000 + for i, piece := range deal.Data.SourceAggregate.Pieces { + var offline bool + if piece.SourceOffline != nil { + offline = true + } + rawSize, err := piece.RawSize() + if err != nil { + return xerrors.Errorf("getting raw size: %w", err) + } + pBatch.Queue(`INSERT INTO market_mk20_pipeline (id, sp_id, contract, client, piece_cid, + piece_size, raw_size, offline, indexing, announce, allocation_id, duration, + piece_aggregation, deal_aggregation, aggr_index, started) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)`, + dealID, spid, ddo.ContractAddress, ddo.Client.String(), piece.PieceCID.String(), + piece.Size, rawSize, offline, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, + 0, data.Format.Aggregate.Type, i, !offline) + if pBatch.Len() > pBatchSize { + res := tx.SendBatch(ctx, pBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 pipeline insert batch: %w", err) + } + pBatch = &pgx.Batch{} + } + } + if pBatch.Len() > 0 { + res := tx.SendBatch(ctx, pBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 pipeline insert batch: %w", err) + } + } + return nil + } + + return xerrors.Errorf("unknown data source type") +} + +func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { + var pieces []MK20PipelinePiece + err := d.db.Select(ctx, &pieces, `SELECT + id, + sp_id, + contract, + piece_index, + piece_cid, + piece_size, + raw_size, + offline, + url, + indexing, + announce, + verified, + allocation_id, + duration, + piece_aggregation, + started, + downloaded, + deal_aggregation, + aggr_index, + agg_task_id, + aggregated, + sector, + reg_seal_proof, + sector_offset, + indexing_created_at, + indexing_task_id, + indexed + FROM + market_mk20_pipeline + WHERE complete = false ORDER BY created_at ASC; + `) + if err != nil { + log.Errorw("failed to get deals from DB", "error", err) + return + } + + for _, piece := range pieces { + err := d.processMk20Pieces(ctx, piece) + if err != nil { + log.Errorw("failed to process deal", "ID", piece.ID, "SP", piece.SPID, "Contract", piece.Contract, "Piece CID", piece.PieceCID, "Piece Size", piece.PieceSize, "error", err) + continue + } + } + +} + +func (d *CurioStorageDealMarket) processMk20Pieces(ctx context.Context, piece MK20PipelinePiece) error { + err := d.downloadMk20Deal(ctx, piece) + if err != nil { + return err + } + + err = d.findOfflineURLMk20Deal(ctx, piece) + if err != nil { + return err + } + + err = d.createCommPMk20Piece(ctx, piece) + if err != nil { + return err + } + + err = d.addDealOffset(ctx, piece) + if err != nil { + return err + } + + return nil +} + +// downloadMk20Deal handles the downloading process of an MK20 pipeline piece by scheduling it in the database and updating its status. +// If the pieces are part of an aggregation deal then we download for short term otherwise we check if piece needs to be indexed. +// If indexing is true then we download for long term to avoid the need to have unsealed copy +func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK20PipelinePiece) error { + if !piece.Downloaded && piece.Started { + _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var refid int64 + err = tx.QueryRow(`SELECT ref_id FROM ( + SELECT unnest(dp.ref_ids) AS ref_id + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 + ) u + JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.complete = TRUE + LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&refid) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) + } + _, err = tx.Exec(` + DELETE FROM parked_piece_refs + WHERE ref_id IN ( + SELECT unnest(dp.ref_ids) + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 + AND dp.piece_cid = $2 + AND dp.piece_size = $3 + ) + AND ref_id != $4; + `, piece.ID, piece.PieceCID, piece.PieceSize, refid) + if err != nil { + return false, xerrors.Errorf("failed to delete parked piece refs: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", refid), + } + + _, err = tx.Exec(`UPDATE market_mk20_pipeline SET downloaded = TRUE, url = $1 + WHERE id = $2 + AND piece_cid = $3 + AND piece_size = $4`, + pieceIDUrl.String(), piece.ID, piece.PieceCID, piece.PieceSize) + if err != nil { + return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) + } + piece.Downloaded = true + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return xerrors.Errorf("failed to schedule the deal for download: %w", err) + } + } + return nil +} + +// findOfflineURLMk20Deal find the URL for offline piece. In MK20, we don't work directly with remote pieces, we download them +// locally and then decide to aggregate, long term or remove them +func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, piece MK20PipelinePiece) error { + if piece.Offline && !piece.Downloaded && !piece.Started { + comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var updated bool + err = tx.QueryRow(` + WITH offline_match AS ( + SELECT url, headers, raw_size + FROM market_mk20_offline_urls + WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 + ), + existing_piece AS ( + SELECT id AS piece_id + FROM parked_pieces + WHERE piece_cid = $2 AND piece_padded_size = $3 + ), + inserted_piece AS ( + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + SELECT $2, $3, o.raw_size, NOT (p.deal_aggregation > 0) + FROM offline_match o, market_mk20_pipeline p + WHERE p.id = $1 AND p.piece_cid = $2 AND p.piece_size = $3 + AND NOT EXISTS (SELECT 1 FROM existing_piece) + RETURNING id AS piece_id + ), + selected_piece AS ( + SELECT piece_id FROM existing_piece + UNION ALL + SELECT piece_id FROM inserted_piece + ), + inserted_refs AS ( + INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + SELECT + s.piece_id, + o.url, + o.headers, + NOT (p.deal_aggregation > 0) + FROM selected_piece s + JOIN offline_match o ON true + JOIN market_mk20_pipeline p ON p.id = $1 AND p.piece_cid = $2 AND p.piece_size = $3 + RETURNING ref_id + ), + upsert_pipeline AS ( + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + SELECT $1, $2, $3, array_agg(ref_id) + FROM inserted_refs + ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + SET ref_ids = ( + SELECT array( + SELECT DISTINCT unnest(dp.ref_ids) || unnest(EXCLUDED.ref_ids) + ) + ) + FROM market_mk20_download_pipeline dp + WHERE dp.id = EXCLUDED.id AND dp.piece_cid = EXCLUDED.piece_cid AND dp.piece_size = EXCLUDED.piece_size + RETURNING id + ), + mark_started AS ( + UPDATE market_mk20_pipeline + SET started = TRUE + WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 + AND EXISTS (SELECT 1 FROM offline_match) + RETURNING id + ) + SELECT EXISTS (SELECT 1 FROM mark_started); + `, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&updated) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("failed to update the pipeline for deal %s: %w", piece.ID, err) + } + } + + if updated { + return true, nil + } + + // Check if We can find the URL for this piece on remote servers + for rUrl, headers := range d.urls { + // Create a new HTTP request + urlString := fmt.Sprintf("%s?id=%s", rUrl, piece.PieceCID) + req, err := http.NewRequest(http.MethodHead, urlString, nil) + if err != nil { + return false, xerrors.Errorf("error creating request: %w", err) + } + + req.Header = headers + + // Create a client and make the request + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Do(req) + if err != nil { + return false, xerrors.Errorf("error making GET request: %w", err) + } + + // Check the response code for 404 + if resp.StatusCode != http.StatusOK { + if resp.StatusCode != 404 { + return false, xerrors.Errorf("not ok response from HTTP server: %s", resp.Status) + } + continue + } + + hdrs, err := json.Marshal(headers) + if err != nil { + return false, xerrors.Errorf("marshaling headers: %w", err) + } + + rawSizeStr := resp.Header.Get("Content-Length") + if rawSizeStr == "" { + continue + } + rawSize, err := strconv.ParseInt(rawSizeStr, 10, 64) + if err != nil { + return false, xerrors.Errorf("failed to parse the raw size: %w", err) + } + + if rawSize != piece.RawSize { + continue + } + + if abi.PaddedPieceSize(piece.PieceSize) != padreader.PaddedSize(uint64(rawSize)).Padded() { + continue + } + + _, err = tx.Exec(`WITH pipeline_piece AS ( + SELECT id, piece_cid, piece_size, deal_aggregation + FROM market_mk20_pipeline + WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 + ), + existing_piece AS ( + SELECT id AS piece_id + FROM parked_pieces + WHERE piece_cid = $2 AND piece_padded_size = $3 + ), + inserted_piece AS ( + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + SELECT $2, $3, $4, NOT (p.deal_aggregation > 0) + FROM pipeline_piece p + WHERE NOT EXISTS (SELECT 1 FROM existing_piece) + RETURNING id AS piece_id + ), + selected_piece AS ( + SELECT piece_id FROM existing_piece + UNION ALL + SELECT piece_id FROM inserted_piece + ), + inserted_ref AS ( + INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + SELECT + s.piece_id, + $5, + $6, + NOT (p.deal_aggregation > 0) + FROM selected_piece s + JOIN pipeline_piece p ON true + RETURNING ref_id + ), + upsert_pipeline AS ( + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + SELECT $1, $2, $3, array_agg(ref_id) + FROM inserted_ref + ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + SET ref_ids = ( + SELECT array( + SELECT DISTINCT unnest(dp.ref_ids) || unnest(EXCLUDED.ref_ids) + ) + ) + FROM market_mk20_download_pipeline dp + WHERE dp.id = EXCLUDED.id AND dp.piece_cid = EXCLUDED.piece_cid AND dp.piece_size = EXCLUDED.piece_size + ), + mark_started AS ( + UPDATE market_mk20_pipeline + SET started = TRUE + WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND started = FALSE + )`, piece.ID, piece.PieceCID, piece.PieceSize, rUrl, hdrs, rawSize) + if err != nil { + return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) + } + + return true, nil + } + return false, nil + + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("deal %s: %w", piece.ID, err) + } + + if comm { + log.Infow("URL attached for offline deal piece", "deal piece", piece) + } + } + + return nil +} + +// createCommPMk20Piece handles the creation of a CommP task for an MK20 pipeline piece, updating its status based on piece attributes. +func (d *CurioStorageDealMarket) createCommPMk20Piece(ctx context.Context, piece MK20PipelinePiece) error { + if piece.Downloaded && !piece.AfterCommp && piece.CommTaskID == nil { + // Skip commP is configured to do so + if d.cfg.Market.StorageMarketConfig.MK12.SkipCommP { + _, err := d.db.Exec(ctx, `UPDATE market_mk20_pipeline SET after_commp = TRUE, commp_task_id = NULL + WHERE id = $1 + AND sp_id = $2 + AND piece_cid = $3 + AND piece_size = $4 + AND raw_size = $5 + AND aggr_index = $6 + AND downloaded = TRUE + AND after_commp = FALSE`, piece.ID, piece.SPID, piece.PieceCID, piece.PieceSize, piece.RawSize, piece.AggregationIndex) + if err != nil { + return xerrors.Errorf("marking piece as after commP: %w", err) + } + log.Infow("commP skipped successfully", "deal piece", piece) + return nil + } + + if d.adders[pollerCommP].IsSet() { + d.adders[pollerCommP].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, err error) { + // update + n, err := tx.Exec(`UPDATE market_mk20_pipeline SET commp_task_id = $1 + WHERE id = $2 + AND sp_id = $3 + AND piece_cid = $4 + AND piece_size = $5 + AND raw_size = $6 + AND aggr_index = $7 + AND downloaded = TRUE + AND after_commp = FALSE + AND commp_task_id = NULL`, id, piece.ID, piece.SPID, piece.PieceCID, piece.PieceSize, piece.RawSize, piece.AggregationIndex) + if err != nil { + return false, xerrors.Errorf("creating commP task for deal piece: %w", err) + } + + // commit only if we updated the piece + return n > 0, nil + }) + log.Infow("commP task created successfully", "deal piece", piece) + } + + return nil + } + return nil +} + +func (d *CurioStorageDealMarket) addDealOffset(ctx context.Context, piece MK20PipelinePiece) error { + // Get the deal offset if sector has started sealing + if piece.Sector != nil && piece.RegSealProof == nil { + _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + type pieces struct { + Cid string `db:"piece_cid"` + Size abi.PaddedPieceSize `db:"piece_size"` + Index int64 `db:"piece_index"` + } + + var pieceList []pieces + err = tx.Select(&pieceList, `SELECT piece_cid, piece_size, piece_index + FROM sectors_sdr_initial_pieces + WHERE sp_id = $1 AND sector_number = $2 + + UNION ALL + + SELECT piece_cid, piece_size, piece_index + FROM sectors_snap_initial_pieces + WHERE sp_id = $1 AND sector_number = $2 + + ORDER BY piece_index ASC;`, piece.SPID, piece.Sector) + if err != nil { + return false, xerrors.Errorf("getting pieces for sector: %w", err) + } + + if len(pieceList) == 0 { + // Sector might be waiting for more deals + return false, nil + } + + var offset abi.UnpaddedPieceSize + + for _, p := range pieceList { + _, padLength := proofs.GetRequiredPadding(offset.Padded(), p.Size) + offset += padLength.Unpadded() + if p.Cid == piece.PieceCID && p.Size == abi.PaddedPieceSize(piece.PieceSize) { + n, err := tx.Exec(`UPDATE market_mk20_pipeline SET sector_offset = $1 WHERE id = $2 AND sector = $3 AND sector_offset IS NULL`, offset.Padded(), piece.ID, piece.Sector) + if err != nil { + return false, xerrors.Errorf("updating deal offset: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 deal, updated %d", n) + } + offset += p.Size.Unpadded() + return true, nil + } + + } + return false, xerrors.Errorf("failed to find deal offset for piece %s", piece.PieceCID) + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("failed to get deal offset: %w", err) + } + } + return nil +} + +func (d *CurioStorageDealMarket) processMK20DealAggregation(ctx context.Context) { + if !d.adders[pollerAggregate].IsSet() { + return + } + + var deals []struct { + ID string `db:"id"` + Count int `db:"count"` + } + + err := d.db.Select(ctx, &deals, `SELECT id, COUNT(*) AS count + FROM market_mk20_pipeline + GROUP BY id + HAVING bool_and(after_commp) + AND bool_and(NOT aggregated) + AND bool_and(agg_task_id IS NULL);`) + if err != nil { + log.Errorf("getting deals to aggregate: %w", err) + return + } + + for _, deal := range deals { + d.adders[pollerAggregate].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, err error) { + n, err := tx.Exec(`UPDATE market_mk20_pipeline SET agg_task_id = $1 + WHERE id = $2 + AND after_commp = TRUE + AND NOT aggregated + AND agg_task_id IS NULL`, id, deal.ID) + if err != nil { + return false, xerrors.Errorf("creating aggregation task for deal: %w", err) + } + return n == deal.Count, nil + }) + } + +} + +func (d *CurioStorageDealMarket) processMK20DealIngestion(ctx context.Context) { + + head, err := d.api.ChainHead(ctx) + if err != nil { + log.Errorf("getting chain head: %w", err) + return + } + + var deals []struct { + ID string `db:"id"` + SPID int64 `db:"sp_id"` + Client string `db:"client"` + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + AllocationID *int64 `db:"allocation_id"` + Duration int64 `db:"duration"` + Url string `db:"url"` + Count int `db:"unassigned_count"` + } + + err = d.db.Select(ctx, &deals, `SELECT + id, + MIN(sp_id) AS sp_id, + MIN(client) AS client, + MIN(piece_cid) AS piece_cid, + MIN(piece_size) AS piece_size, + MIN(raw_size) AS raw_size, + MIN(allocation_id) AS allocation_id, + MIN(duration) AS duration, + MIN(url) AS url, + COUNT(*) AS unassigned_count + FROM market_mk20_pipeline + WHERE aggregated = TRUE AND sector IS NULL + GROUP BY id;`) + if err != nil { + log.Errorf("getting deals for ingestion: %w", err) + return + } + + for _, deal := range deals { + if deal.Count != 1 { + log.Errorf("unexpected count for deal: %s", deal.ID) + continue + } + + pcid, err := cid.Parse(deal.PieceCID) + if err != nil { + log.Errorw("failed to parse aggregate piece cid", "deal", deal, "error", err) + continue + } + + client, err := address.NewFromString(deal.Client) + if err != nil { + log.Errorw("failed to parse client address", "deal", deal, "error", err) + continue + } + + clientId, err := address.IDFromAddress(client) + if err != nil { + log.Errorw("failed to parse client id", "deal", deal, "error", err) + continue + } + + aurl, err := url.Parse(deal.Url) + if err != nil { + log.Errorf("failed to parse aggregate url: %w", err) + continue + } + if aurl.Scheme != "pieceref" { + log.Errorw("aggregate url is not a pieceref: %s", deal) + continue + } + + start := head.Height() + 2*builtin.EpochsInDay + end := start + abi.ChainEpoch(deal.Duration) + var vak *miner.VerifiedAllocationKey + if deal.AllocationID != nil { + alloc, err := d.api.StateGetAllocation(ctx, client, verifreg.AllocationId(*deal.AllocationID), types.EmptyTSK) + if err != nil { + log.Errorw("failed to get allocation", "deal", deal, "error", err) + continue + } + if alloc == nil { + log.Errorw("allocation not found", "deal", deal, "error", err) + continue + } + if alloc.Expiration < start { + log.Errorw("allocation expired", "deal", deal, "error", err) + continue + } + end = start + alloc.TermMin + vak = &miner.VerifiedAllocationKey{ + Client: abi.ActorID(clientId), + ID: verifreg13.AllocationId(*deal.AllocationID), + } + } + + // TODO: Attach notifications + pdi := lpiece.PieceDealInfo{ + DealSchedule: lpiece.DealSchedule{ + StartEpoch: start, + EndEpoch: end, + }, + PieceActivationManifest: &miner.PieceActivationManifest{ + CID: pcid, + Size: abi.PaddedPieceSize(deal.PieceSize), + VerifiedAllocationKey: vak, + }, + } + + maddr, err := address.NewIDAddress(uint64(deal.SPID)) + if err != nil { + log.Errorw("failed to parse miner address", "deal", deal, "error", err) + continue + } + + comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + sector, sp, err := d.pin.AllocatePieceToSector(ctx, tx, maddr, pdi, deal.RawSize, *aurl, nil) + if err != nil { + return false, xerrors.Errorf("failed to allocate piece to sector: %w", err) + } + n, err := tx.Exec(`UPDATE market_mk20_pipeline SET SET sector = $1, reg_seal_proof = $2 WHERE id = $3`, *sector, *sp, deal.ID) + if err != nil { + return false, xerrors.Errorf("failed to update deal: %w", err) + } + return n == 1, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorf("failed to commit transaction: %w", err) + continue + } + if comm { + log.Infow("deal ingested successfully", "deal", deal) + } else { + log.Infow("deal not ingested", "deal", deal) + } + } +} diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index 927e07b07..ff102e898 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/ethereum/go-ethereum/ethclient" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/yugabyte/pgx/v5" @@ -33,6 +34,7 @@ import ( "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/curio/market/mk12" "github.com/filecoin-project/curio/market/mk12/legacytypes" + "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/market/storageingest" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -51,6 +53,7 @@ const ( pollerCommP = iota pollerPSD pollerFindDeal + pollerAggregate numPollers ) @@ -69,6 +72,8 @@ type CurioStorageDealMarket struct { miners map[string][]address.Address api storageMarketAPI MK12Handler *mk12.MK12 + MK20Handler *mk20.MK20 + ethClient *ethclient.Client si paths.SectorIndex urls map[string]http.Header adders [numPollers]promise.Promise[harmonytask.AddTaskFunc] @@ -109,7 +114,7 @@ type MK12Pipeline struct { Offset *int64 `db:"sector_offset"` } -func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector) *CurioStorageDealMarket { +func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector) *CurioStorageDealMarket { moduleMap := make(map[string][]address.Address) moduleMap[mk12Str] = append(moduleMap[mk12Str], miners...) @@ -120,13 +125,14 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * } return &CurioStorageDealMarket{ - cfg: cfg, - db: db, - api: mapi, - miners: moduleMap, - si: si, - urls: urls, - as: as, + cfg: cfg, + db: db, + api: mapi, + miners: moduleMap, + si: si, + urls: urls, + as: as, + ethClient: ethClient, } } @@ -173,6 +179,15 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { d.pin, err = storageingest.NewPieceIngester(ctx, d.db, d.api, miners, d.cfg) } } + if module == mk20Str && d.pin != nil { + if len(miners) == 0 { + return nil + } + d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as) + if err != nil { + return err + } + } } if err != nil { @@ -185,6 +200,15 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { } func (d *CurioStorageDealMarket) runPoller(ctx context.Context) { + // Start thread to insert mk20 DDO deals into pipeline + for module, miners := range d.miners { + if module == mk20Str { + if len(miners) > 0 { + go d.pipelineInsertLoop(ctx) + } + } + } + ticker := time.NewTicker(dealPollerInterval) defer ticker.Stop() @@ -222,6 +246,7 @@ func (d *CurioStorageDealMarket) poll(ctx context.Context) { if module == mk12Str { if len(miners) > 0 { d.processMK12Deals(ctx) + d.processMK20Deals(ctx) } } } diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go new file mode 100644 index 000000000..f6df5810b --- /dev/null +++ b/tasks/storage-market/task_aggregation.go @@ -0,0 +1,344 @@ +package storage_market + +import ( + "context" + "fmt" + "io" + "math/bits" + "net/url" + "os" + "strconv" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-commp-utils/writer" + "github.com/filecoin-project/go-data-segment/datasegment" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/dealdata" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +type AggregateTask struct { + sm *CurioStorageDealMarket + db *harmonydb.DB + sc *ffi.SealCalls + stor paths.StashStore + api headAPI + max int +} + +func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.SealCalls, stor paths.StashStore, api headAPI, max int) *AggregateTask { + return &AggregateTask{ + sm: sm, + db: db, + sc: sc, + stor: stor, + api: api, + max: max, + } +} + +func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var pieces []struct { + Pcid string `db:"piece_cid"` + Psize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + URL string `db:"url"` + ID string `db:"id"` + SpID int64 `db:"sp_id"` + AggrIndex int `db:"aggr_index"` + Aggregated bool `db:"aggregated"` + Aggreation int `db:"deal_aggregation"` + } + + err = a.db.Select(ctx, &pieces, ` + SELECT + url, + headers, + raw_size, + piece_cid, + piece_size, + id, + sp_id, + aggr_index + FROM + market_mk20_pipeline + WHERE + agg_task_id = $1 ORDER BY aggr_index ASC`, taskID) + if err != nil { + return false, xerrors.Errorf("getting piece details: %w", err) + } + + if len(pieces) == 0 { + return false, xerrors.Errorf("no pieces to aggregate for task %d", taskID) + } + + if len(pieces) == 1 { + n, err := a.db.Exec(ctx, `UPDATE market_mk20_pipeline SET aggregated = TRUE, agg_task_id = NULL + WHERE id = $1 + AND agg_task_id = $2`, pieces[0].ID, taskID) + if err != nil { + return false, xerrors.Errorf("updating aggregated piece details in DB: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row updated, got %d", n) + } + log.Infof("skipping aggregation as deal %s only has 1 piece for task %s", pieces[0].ID, taskID) + return true, nil + } + + id := pieces[0].ID + spid := pieces[0].SpID + + ID, err := ulid.Parse(id) + if err != nil { + return false, xerrors.Errorf("parsing deal ID: %w", err) + } + + deal, err := mk20.DealFromDB(ctx, a.db, ID) + if err != nil { + return false, xerrors.Errorf("getting deal details from DB: %w", err) + } + + rawSize, err := deal.Data.RawSize() + if err != nil { + return false, xerrors.Errorf("getting raw size: %w", err) + } + + var pinfos []abi.PieceInfo + var readers []io.Reader + + for _, piece := range pieces { + if piece.Aggregated { + return false, xerrors.Errorf("piece %s for deal %s already aggregated for task %d", piece.Pcid, piece.ID, taskID) + } + if piece.Aggreation != 1 { + return false, xerrors.Errorf("incorrect aggregation value for piece %s for deal %s for task %d", piece.Pcid, piece.ID, taskID) + } + if piece.ID != id || piece.SpID != spid { + return false, xerrors.Errorf("piece details do not match") + } + goUrl, err := url.Parse(piece.URL) + if err != nil { + return false, xerrors.Errorf("parsing data URL: %w", err) + } + if goUrl.Scheme != "pieceref" { + return false, xerrors.Errorf("invalid data URL scheme: %s", goUrl.Scheme) + } + + var reader io.Reader // io.ReadCloser is not supported by padreader + var closer io.Closer + + refNum, err := strconv.ParseInt(goUrl.Opaque, 10, 64) + if err != nil { + return false, xerrors.Errorf("parsing piece reference number: %w", err) + } + + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = a.db.Select(ctx, &pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, refNum) + if err != nil { + return false, xerrors.Errorf("getting pieceID: %w", err) + } + + if len(pieceID) != 1 { + return false, xerrors.Errorf("expected 1 pieceID, got %d", len(pieceID)) + } + + pr, err := a.sc.PieceReader(ctx, pieceID[0].PieceID) + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + + closer = pr + reader = pr + defer func() { + _ = closer.Close() + }() + + pcid, err := cid.Parse(piece.Pcid) + if err != nil { + return false, xerrors.Errorf("parsing piece cid: %w", err) + } + + pinfos = append(pinfos, abi.PieceInfo{ + Size: abi.PaddedPieceSize(piece.Psize), + PieceCID: pcid, + }) + + pReader, _ := padreader.New(reader, uint64(piece.RawSize)) + readers = append(readers, pReader) + } + + _, aggregatedRawSize, err := datasegment.ComputeDealPlacement(pinfos) + if err != nil { + return false, xerrors.Errorf("computing aggregated piece size: %w", err) + } + + overallSize := abi.PaddedPieceSize(aggregatedRawSize) + // we need to make this the 'next' power of 2 in order to have space for the index + next := 1 << (64 - bits.LeadingZeros64(uint64(overallSize+256))) + + aggr, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), pinfos) + if err != nil { + return false, xerrors.Errorf("creating aggregate: %w", err) + } + + outR, err := aggr.AggregateObjectReader(readers) + if err != nil { + return false, xerrors.Errorf("aggregating piece readers: %w", err) + } + + w := &writer.Writer{} + + // Function to write data into StashStore and calculate commP + writeFunc := func(f *os.File) error { + multiWriter := io.MultiWriter(w, f) + + // Copy data from limitedReader to multiWriter + n, err := io.CopyBuffer(multiWriter, outR, make([]byte, writer.CommPBuf)) + if err != nil { + return fmt.Errorf("failed to read and write aggregated piece data: %w", err) + } + + if n != int64(rawSize) { + return fmt.Errorf("number of bytes written to CommP writer %d not equal to the file size %d", n, aggregatedRawSize) + } + + return nil + } + + stashID, err := a.stor.StashCreate(ctx, int64(next), writeFunc) + if err != nil { + return false, xerrors.Errorf("stashing aggregated piece data: %w", err) + } + + calculatedCommp, err := w.Sum() + if err != nil { + return false, xerrors.Errorf("computing commP failed: %w", err) + } + + if !calculatedCommp.PieceCID.Equals(deal.Data.PieceCID) { + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", calculatedCommp.PieceCID.String(), deal.Data.PieceCID.String()) + } + + if calculatedCommp.PieceSize != deal.Data.Size { + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", calculatedCommp.PieceSize, deal.Data.Size) + } + + comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var parkedPieceID int64 + + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, TRUE) RETURNING id + `, calculatedCommp.PieceCID.String(), calculatedCommp.PieceSize, rawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } + + // Create a piece ref with data_url being "stashstore://" + // Get StashURL + stashURL, err := a.stor.StashURL(stashID) + if err != nil { + return false, fmt.Errorf("failed to get stash URL: %w", err) + } + + // Change scheme to "custore" + stashURL.Scheme = dealdata.CustoreScheme + dataURL := stashURL.String() + + var pieceRefID int64 + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, dataURL).Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", pieceRefID), + } + + // Replace the pipeline piece with a new aggregated piece + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline WHERE id = $1`, id) + if err != nil { + return false, fmt.Errorf("failed to delete pipeline pieces: %w", err) + } + + ddo := deal.Products.DDOV1 + data := deal.Data + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, piece_size, raw_size, url, + offline, indexing, announce, allocation_id, duration, + piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, TRUE, TRUE, TRUE, TRUE)`, + id, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(data.SourceHTTP.RawSize), pieceIDUrl.String(), + false, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, + data.Format.Aggregate.Type, data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting aggregated piece in mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting aggregated piece in mk20 pipeline: %d rows affected", n) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("saving aggregated piece details to DB: %w", err) + } + + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") + } + return true, nil +} + +func (a *AggregateTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + // If no local pieceRef was found then just return first TaskID + return &ids[0], nil +} + +func (a *AggregateTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(a.max), + Name: "AggregateDeals", + Cost: resources.Resources{ + Cpu: 1, + Ram: 4 << 30, + }, + MaxFailures: 3, + } +} + +func (a *AggregateTask) Adder(taskFunc harmonytask.AddTaskFunc) { + a.sm.adders[pollerAggregate].Set(taskFunc) +} + +var _ = harmonytask.Reg(&AggregateTask{}) +var _ harmonytask.TaskInterface = &AggregateTask{} diff --git a/tasks/storage-market/task_commp.go b/tasks/storage-market/task_commp.go index 5a518011b..fb828dec3 100644 --- a/tasks/storage-market/task_commp.go +++ b/tasks/storage-market/task_commp.go @@ -50,32 +50,80 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done ctx := context.Background() var pieces []struct { - Pcid string `db:"piece_cid"` - Psize int64 `db:"piece_size"` - UUID string `db:"uuid"` - URL *string `db:"url"` - Headers json.RawMessage `db:"headers"` - RawSize int64 `db:"raw_size"` + Pcid string `db:"piece_cid"` + Psize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + URL *string `db:"url"` + Headers json.RawMessage `db:"headers"` + UUID *string `db:"uuid"` // Nullable because it only exists in market_mk12_deal_pipeline + ID *string `db:"id"` // Nullable because it only exists in market_mk20_pipeline + IDType *int `db:"id_type"` + SpID *int64 `db:"sp_id"` + Contract *string `db:"contract"` + PieceIndex *int `db:"piece_index"` + MK12Piece bool `db:"mk12_source_table"` } - err = c.db.Select(ctx, &pieces, `SELECT uuid, url, headers, raw_size, piece_cid, piece_size - FROM market_mk12_deal_pipeline WHERE commp_task_id = $1`, taskID) - + err = c.db.Select(ctx, &pieces, `SELECT + uuid, + url, + headers, + raw_size, + piece_cid, + piece_size, + NULL AS id, + NULL AS id_type, + NULL AS sp_id, + NULL AS contract, + NULL AS piece_index, + TRUE AS mk12_source_table + FROM + market_mk12_deal_pipeline + WHERE + commp_task_id = $1 + + UNION ALL + + SELECT + NULL AS uuid, + url, + headers, + raw_size, + piece_cid, + piece_size, + id, + id_type, + sp_id, + contract, + piece_index, + FALSE AS mk12_source_table + FROM + market_mk20_pipeline + WHERE + commp_task_id = $1`, taskID) if err != nil { return false, xerrors.Errorf("getting piece details: %w", err) } - if len(pieces) != 1 { return false, xerrors.Errorf("expected 1 piece, got %d", len(pieces)) } piece := pieces[0] - expired, err := checkExpiry(ctx, c.db, c.api, piece.UUID, c.sm.pin.GetExpectedSealDuration()) - if err != nil { - return false, xerrors.Errorf("deal %s expired: %w", piece.UUID, err) - } - if expired { - return true, nil + if piece.MK12Piece { + if piece.UUID == nil { + return false, xerrors.Errorf("expected UUID to be non-null for mk12 piece") + } + expired, err := checkExpiry(ctx, c.db, c.api, *piece.UUID, c.sm.pin.GetExpectedSealDuration()) + if err != nil { + return false, xerrors.Errorf("deal %s expired: %w", *piece.UUID, err) + } + if expired { + return true, nil + } + } else { + if piece.ID == nil || piece.IDType == nil || piece.SpID == nil || piece.Contract == nil || piece.PieceIndex == nil { + return false, xerrors.Errorf("expected ID, IDType, SpID, Contract, PieceIndex to be non-null for mk20 piece") + } } if piece.URL != nil { @@ -196,7 +244,25 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", pcid, calculatedCommp.PieceCID) } - n, err := c.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET after_commp = TRUE, psd_wait_time = NOW(), commp_task_id = NULL WHERE commp_task_id = $1`, taskID) + var n int + + if piece.MK12Piece { + n, err = c.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET after_commp = TRUE, psd_wait_time = NOW(), commp_task_id = NULL WHERE commp_task_id = $1`, taskID) + } else { + n, err = c.db.Exec(ctx, `UPDATE market_mk20_pipeline SET after_commp = TRUE, commp_task_id = $9 + WHERE id = $1 + AND id_type = $2 + AND sp_id = $3 + AND contract = $4 + AND piece_cid = $5 + AND piece_size = $6 + AND raw_size = $7 + AND piece_index = $8 + AND downloaded = TRUE + AND after_commp = FALSE`, + *piece.ID, *piece.IDType, *piece.SpID, *piece.Contract, piece.Pcid, piece.Psize, piece.RawSize, *piece.PieceIndex, taskID) + } + if err != nil { return false, xerrors.Errorf("store commp success: updating deal pipeline: %w", err) } @@ -207,7 +273,11 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return true, nil } - return false, xerrors.Errorf("failed to find URL for the piece %s in the db", piece.Pcid) + if piece.MK12Piece { + return false, xerrors.Errorf("failed to find URL for the piece %s in the db", piece.Pcid) + } + + return false, xerrors.Errorf("failed to find URL for the mk20 deal piece with id %s, idType %d, SP %d, Contract %s, Index %d and CID %s in the db", *piece.ID, *piece.IDType, *piece.SpID, *piece.Contract, *piece.PieceIndex, piece.Pcid) } @@ -218,20 +288,13 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task // ParkPiece should be scheduled on same node which has the piece // Remote HTTP ones can be scheduled on any node - if true { - // TODO make this a setting - id := ids[0] - return &id, nil - } - ctx := context.Background() var tasks []struct { - TaskID harmonytask.TaskID `db:"commp_task_id"` - SpID int64 `db:"sp_id"` - SectorNumber int64 `db:"sector_number"` - StorageID string `db:"storage_id"` - Url *string `db:"url"` + TaskID harmonytask.TaskID `db:"commp_task_id"` + SpID int64 `db:"sp_id"` + StorageID string `db:"storage_id"` + Url *string `db:"url"` } indIDs := make([]int64, len(ids)) @@ -240,9 +303,26 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task } comm, err := c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - err = tx.Select(&tasks, ` - SELECT commp_task_id, sp_id, sector_number, url FROM market_mk12_deal_pipeline - WHERE commp_task_id = ANY ($1)`, indIDs) + err = tx.Select(&tasks, `SELECT + commp_task_id, + sp_id, + url + FROM + market_mk12_deal_pipeline + WHERE + commp_task_id = ANY ($1) + + UNION ALL + + SELECT + commp_task_id, + sp_id, + url + FROM + market_mk20_pipeline + WHERE + commp_task_id = ANY ($1); + `, indIDs) if err != nil { return false, xerrors.Errorf("failed to get deal details from DB: %w", err) } @@ -286,6 +366,7 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task } } } + return true, nil }, harmonydb.OptionRetry()) From acbd66beebe6c44443dcb0a2ba4cdb0e8121b66b Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 14 May 2025 18:09:06 +0400 Subject: [PATCH 02/55] PUT endpoint for deals --- cmd/curio/tasks/tasks.go | 2 +- .../harmonydb/sql/20250505-market_mk20.sql | 28 +- market/mk20/ddo_v1.go | 70 ++-- market/mk20/http/http.go | 77 ++-- market/mk20/mk20.go | 47 ++- market/mk20/mk20_utils.go | 332 ++++++++++++++++++ market/mk20/types.go | 124 ++++++- market/mk20/utils.go | 167 ++++++--- tasks/storage-market/mk20.go | 2 +- tasks/storage-market/storage_market.go | 12 +- 10 files changed, 725 insertions(+), 136 deletions(-) create mode 100644 market/mk20/mk20_utils.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index f70d8b082..f1ed36a2c 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -249,7 +249,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan var dm *storage_market.CurioStorageDealMarket if cfg.Subsystems.EnableDealMarket { // Main market poller should run on all nodes - dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as) + dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as, lstor) err := dm.StartMarket(ctx) if err != nil { return nil, err diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 67e08b85b..f9881d7d1 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -15,7 +15,9 @@ CREATE TABLE market_mk20_deal ( source_offline JSONB NOT NULL DEFAULT 'null', ddov1 JSONB NOT NULL DEFAULT 'null', - market_deal_id TEXT DEFAULT NULL + market_deal_id TEXT DEFAULT NULL, + + error TEXT DEFAULT NULL ); CREATE TABLE market_mk20_pipeline ( @@ -63,7 +65,10 @@ CREATE TABLE market_mk20_pipeline ( ); CREATE TABLE market_mk20_pipeline_waiting ( - id TEXT NOT NULL PRIMARY KEY + id TEXT PRIMARY KEY, + waiting_for_data BOOLEAN DEFAULT FALSE, + started_put BOOLEAN DEFAULT FALSE, + start_time TIMESTAMPZ DEFAULT NULL ); CREATE TABLE market_mk20_download_pipeline ( @@ -81,12 +86,29 @@ CREATE TABLE market_mk20_offline_urls ( url TEXT NOT NULL, headers jsonb NOT NULL DEFAULT '{}', raw_size BIGINT NOT NULL, + PRIMARY KEY (id, piece_cid, piece_size), CONSTRAINT market_mk20_offline_urls_id_fk FOREIGN KEY (id) - REFERENCES market_mk20_deal_pipeline (id) + REFERENCES market_mk20_pipeline (id) ON DELETE CASCADE, CONSTRAINT market_mk20_offline_urls_id_unique UNIQUE (id) ); +CREATE TABLE market_mk20_products ( + name TEXT PRIMARY KEY, + enabled BOOLEAN DEFAULT TRUE +); + +CREATE TABLE market_mk20_data_source ( + name TEXT PRIMARY KEY, + enabled BOOLEAN DEFAULT TRUE +); + +INSERT INTO market_mk20_products (name, enabled) VALUES ('ddov1', TRUE); +INSERT INTO market_mk20_data_source (name, enabled) VALUES ('http', TRUE); +INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); +INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); +INSERT INTO market_mk20_data_source (name, enabled) VALUES ('put', TRUE); + diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index b1b4a0633..62026ec1c 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "strings" "github.com/ethereum/go-ethereum" @@ -64,63 +65,68 @@ type DDOV1 struct { AnnounceToIPNI bool `json:"announcetoinpni"` } -func (d *DDOV1) Validate() error { +func (d *DDOV1) Validate(dbProducts []dbProduct) (int, error) { + code, err := d.IsEnabled(dbProducts) + if err != nil { + return code, err + } + if d.Provider == address.Undef || d.Provider.Empty() { - return xerrors.Errorf("provider address is not set") + return ErrProductValidationFailed, xerrors.Errorf("provider address is not set") } if d.Client == address.Undef || d.Client.Empty() { - return xerrors.Errorf("client address is not set") + return ErrProductValidationFailed, xerrors.Errorf("client address is not set") } if d.PieceManager == address.Undef || d.PieceManager.Empty() { - return xerrors.Errorf("piece manager address is not set") + return ErrProductValidationFailed, xerrors.Errorf("piece manager address is not set") } if d.AllocationId != nil { if *d.AllocationId == verifreg.NoAllocationID { - return xerrors.Errorf("incorrect allocation id") + return ErrProductValidationFailed, xerrors.Errorf("incorrect allocation id") } } if d.AllocationId == nil { if d.Duration < 518400 { - return xerrors.Errorf("duration must be at least 518400") + return ErrDurationTooShort, xerrors.Errorf("duration must be at least 518400") } } if d.ContractAddress == "" { - return xerrors.Errorf("contract address is not set") + return ErrProductValidationFailed, xerrors.Errorf("contract address is not set") } if d.ContractAddress[0:2] != "0x" { - return xerrors.Errorf("contract address must start with 0x") + return ErrProductValidationFailed, xerrors.Errorf("contract address must start with 0x") } if d.ContractDealIDMethodParams == nil { - return xerrors.Errorf("contract deal id method params is not set") + return ErrProductValidationFailed, xerrors.Errorf("contract deal id method params is not set") } if d.ContractDealIDMethod == "" { - return xerrors.Errorf("contract deal id method is not set") + return ErrProductValidationFailed, xerrors.Errorf("contract deal id method is not set") } - return nil + return Ok, nil } -func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, error) { +func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, int, error) { var abiStr string err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return "", UnknowContract + return "", ErrMarketNotEnabled, UnknowContract } - return "", xerrors.Errorf("getting abi: %w", err) + return "", http.StatusInternalServerError, xerrors.Errorf("getting abi: %w", err) } parsedABI, err := eabi.JSON(strings.NewReader(abiStr)) if err != nil { - panic(err) + return "", http.StatusInternalServerError, xerrors.Errorf("parsing abi: %w", err) } to := common.HexToAddress(d.ContractAddress) @@ -128,18 +134,18 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Get the method method, exists := parsedABI.Methods[d.ContractDealIDMethod] if !exists { - return "", fmt.Errorf("method %s not found in ABI", d.ContractDealIDMethod) + return "", http.StatusInternalServerError, fmt.Errorf("method %s not found in ABI", d.ContractDealIDMethod) } // Enforce method must take exactly one `bytes` parameter if len(method.Inputs) != 1 || method.Inputs[0].Type.String() != "bytes" { - return "", fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) + return "", http.StatusInternalServerError, fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) } // ABI-encode method call with input callData, err := parsedABI.Pack(method.Name, d.ContractDealIDMethodParams) if err != nil { - return "", fmt.Errorf("failed to encode call data: %w", err) + return "", http.StatusInternalServerError, fmt.Errorf("failed to encode call data: %w", err) } // Build call message @@ -151,13 +157,35 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Call contract output, err := eth.CallContract(ctx, msg, nil) if err != nil { - return "", fmt.Errorf("eth_call failed: %w", err) + return "", http.StatusInternalServerError, fmt.Errorf("eth_call failed: %w", err) } // Decode return value (assume string) var result string if err := parsedABI.UnpackIntoInterface(&result, method.Name, output); err != nil { - return "", fmt.Errorf("decode result: %w", err) + return "", http.StatusInternalServerError, fmt.Errorf("decode result: %w", err) + } + + if result == "" { + return "", ErrDealRejectedByMarket, fmt.Errorf("empty result from contract") + } + + return result, Ok, nil +} + +func (d *DDOV1) ProductName() ProductName { + return ProductNameDDOV1 +} + +func (d *DDOV1) IsEnabled(dbProducts []dbProduct) (int, error) { + name := string(d.ProductName()) + for _, p := range dbProducts { + if p.Name == name { + if p.Enabled { + return Ok, nil + } + return ErrProductNotEnabled, xerrors.Errorf("product %s is not enabled on the provider", name) + } } - return result, nil + return ErrUnsupportedProduct, xerrors.Errorf("product %s is not supported on the provider", name) } diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index eea888af6..e29371577 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -6,11 +6,13 @@ import ( "fmt" "io" "net/http" + "strings" "time" "github.com/go-chi/chi/v5" "github.com/go-chi/httprate" logging "github.com/ipfs/go-log/v2" + "github.com/oklog/ulid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -23,6 +25,8 @@ import ( var log = logging.Logger("mk20httphdlr") +const maxPutBodySize int64 = 64 << 30 // 64 GiB + type MK20DealHandler struct { cfg *config.CurioConfig db *harmonydb.DB // Replace with your actual DB wrapper if different @@ -53,6 +57,7 @@ func Router(mdh *MK20DealHandler) http.Handler { //mux.Get("/ask", mdh.mk20ask) mux.Get("/status", mdh.mk20status) mux.Get("/contracts", mdh.mk20supportedContracts) + mux.Put("/data", mdh.mk20UploadDealData) return mux } @@ -93,41 +98,30 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { } func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { - ct := r.Header.Get("Content-Type") - var request mk20.DealStatusRequest - - if ct != "application/json" { - log.Errorf("invalid content type: %s", ct) + // Extract id from the URL + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) w.WriteHeader(http.StatusBadRequest) return } - defer r.Body.Close() - body, err := io.ReadAll(r.Body) + id, err := ulid.Parse(idStr) if err != nil { - log.Errorf("error reading request body: %s", err) - w.WriteHeader(http.StatusBadRequest) - } - err = json.Unmarshal(body, &request) - if err != nil { - log.Errorf("error unmarshaling json: %s", err) + log.Errorw("invalid id in url", "id", idStr, "err", err) w.WriteHeader(http.StatusBadRequest) return } - result, err := mdh.dm.MK20Handler.DealStatus(context.Background(), &request) - if err != nil { - log.Errorw("failed to get deal status", "id", request.Identifier, - "idType", request.IdentifierType, - "contractAddress", request.ContractAddress, "err", err) - w.WriteHeader(http.StatusInternalServerError) + result := mdh.dm.MK20Handler.DealStatus(context.Background(), id) + + if result.HTTPCode != http.StatusOK { + w.WriteHeader(result.HTTPCode) return } - resp, err := json.Marshal(result) + resp, err := json.Marshal(result.Response) if err != nil { - log.Errorw("failed to marshal deal status response", "id", request.Identifier, - "idType", request.IdentifierType, - "contractAddress", request.ContractAddress, "err", err) + log.Errorw("failed to marshal deal status response", "id", idStr, "err", err) w.WriteHeader(http.StatusInternalServerError) return } @@ -135,9 +129,7 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, err = w.Write(resp) if err != nil { - log.Errorw("failed to write deal status response", "id", request.Identifier, - "idType", request.IdentifierType, - "contractAddress", request.ContractAddress, "err", err) + log.Errorw("failed to write deal status response", "id", idStr, "err", err) } } @@ -163,3 +155,36 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt log.Errorw("failed to write supported contracts", "err", err) } } + +func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Request) { + // Extract id from the URL + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + w.WriteHeader(http.StatusBadRequest) + return + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + // Check Content-Type + ct := r.Header.Get("Content-Type") + if ct == "" || !strings.HasPrefix(ct, "application/octet-stream") { + http.Error(w, "invalid or missing Content-Type", http.StatusUnsupportedMediaType) + return + } + + // validate Content-Length + if r.ContentLength <= 0 || r.ContentLength > maxPutBodySize { + http.Error(w, fmt.Sprintf("invalid Content-Length: %d", r.ContentLength), http.StatusRequestEntityTooLarge) + return + } + + // Stream directly to execution logic + mdh.dm.MK20Handler.HandlePutRequest(context.Background(), id, r.Body, w) +} diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 918c47e66..410d63b7a 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -42,9 +42,10 @@ type MK20 struct { cfg *config.CurioConfig sm map[address.Address]abi.SectorSize as *multictladdr.MultiAddressSelector + stor paths.StashStore } -func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector) (*MK20, error) { +func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, stor paths.StashStore) (*MK20, error) { ctx := context.Background() sm := make(map[address.Address]abi.SectorSize) @@ -68,23 +69,24 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI cfg: cfg, sm: sm, as: as, + stor: stor, }, nil } func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { // Validate the DataSource TODO: Add error code to validate - valid, err := deal.Validate() + code, err := deal.Validate(m.db) if err != nil { - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, - Reason: "Invalid data source", + log.Errorw("deal rejected", "deal", deal, "error", err) + ret := &ProviderDealRejectionInfo{ + HTTPCode: code, } - } - if !valid { - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, - Reason: "Invalid data source", + if code == http.StatusInternalServerError { + ret.Reason = "Internal server error" + } else { + ret.Reason = err.Error() } + return ret } return m.processDDODeal(ctx, deal) @@ -101,12 +103,18 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje return rejection } - id, err := deal.Products.DDOV1.GetDealID(ctx, m.db, m.ethClient) + id, code, err := deal.Products.DDOV1.GetDealID(ctx, m.db, m.ethClient) if err != nil { log.Errorw("error getting deal ID", "deal", deal, "error", err) - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + ret := &ProviderDealRejectionInfo{ + HTTPCode: code, } + if code == http.StatusInternalServerError { + ret.Reason = "Internal server error" + } else { + ret.Reason = err.Error() + } + return ret } // TODO: Backpressure, client filter @@ -123,7 +131,13 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje if n != 1 { return false, fmt.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + if deal.Data.SourceHttpPut != nil { + _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id, waiting_for_data) VALUES ($1, TRUE) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + + } else { + _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + } + if err != nil { return false, xerrors.Errorf("adding deal to waiting pipeline: %w", err) } @@ -238,11 +252,6 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe return nil, nil } -func (m *MK20) DealStatus(ctx context.Context, statusRequest *DealStatusRequest) (*DealStatusResponse, error) { - // TODO: implement this - return nil, nil -} - // To be used later for when data source is minerID //func validateMinerAddresses(madrs []abi.Multiaddrs, pcid cid.Cid, psize abi.PaddedPieceSize, rawSize int64) bool { // var surls []*url.URL diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go new file mode 100644 index 000000000..618891af9 --- /dev/null +++ b/market/mk20/mk20_utils.go @@ -0,0 +1,332 @@ +package mk20 + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/dealdata" + "github.com/filecoin-project/go-address" + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" +) + +const PutGracePeriod = time.Hour + +func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { + // Check if we ever accepted this deal + + var dealError sql.NullString + + err := m.db.QueryRow(ctx, `SELECT error FROM market_mk20_pipeline WHERE id = $1)`, id.String()).Scan(&dealError) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return &DealStatus{ + HTTPCode: http.StatusNotFound, + } + } + log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + if dealError.Valid { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateFailed, + ErrorMsg: dealError.String, + }, + } + } + + var waitingForPipeline bool + err = m.db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_pipeline_waiting WHERE id = $1)`, id.String()).Scan(&waitingForPipeline) + if err != nil { + log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + if waitingForPipeline { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateAccepted, + }, + } + } + + var pdeals []struct { + Sector *int `db:"sector"` + Sealed bool `db:"sealed"` + Indexed bool `db:"indexed"` + } + + err = m.db.Select(ctx, &pdeals, `SELECT + sector, + sealed, + indexed + FROM + market_mk20_pipeline + WHERE + id = $1`, id.String()) + + if err != nil { + log.Errorw("failed to query the db for deal pipeline status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + + if len(pdeals) > 1 { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateProcessing, + }, + } + } + + // If deal is still in pipeline + if len(pdeals) == 1 { + pdeal := pdeals[0] + if pdeal.Sector == nil { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateProcessing, + }, + } + } + if !pdeal.Sealed { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateSealing, + }, + } + } + if !pdeal.Indexed { + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateIndexing, + }, + } + } + } + + return &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealStatusResponse{ + State: DealStateComplete, + }, + } +} + +func (m *MK20) HandlePutRequest(ctx context.Context, id ulid.ULID, data io.ReadCloser, w http.ResponseWriter) { + defer data.Close() + + var waitingDeal []struct { + Started bool `db:"started_put"` + StartTime *time.Time `db:"start_time"` + } + + err := m.db.Select(ctx, &waitingDeal, `SELECT started_put, start_time from market_mk20_pipeline_waiting + WHERE waiting_for_data = TRUE AND id = $1)`, id.String()) + + if err != nil { + log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if len(waitingDeal) == 0 { + http.Error(w, "", http.StatusNotFound) + } + + if waitingDeal[0].Started && waitingDeal[0].StartTime.Add(PutGracePeriod).Before(time.Now()) { + http.Error(w, "another /PUT request is in progress for this deal", http.StatusConflict) + } + + // TODO: Rethink how to ensure only 1 process per deal for /PUT + + deal, err := DealFromDB(ctx, m.db, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id.String(), "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + rawSize, err := deal.Data.RawSize() + if err != nil { + log.Errorw("failed to get raw size of deal", "deal", id.String(), "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + n, err := m.db.Exec(ctx, `UPDATE market_mk20_pipeline_waiting SET started_put = TRUE, start_time = NOW() WHERE id = $1`, id.String()) + if err != nil { + log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if n != 1 { + log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + failed := true + + defer func() { + if failed { + _, err = m.db.Exec(ctx, `UPDATE market_mk20_pipeline_waiting SET started_put = FALSE, start_time = NULL WHERE id = $1`, id.String()) + if err != nil { + log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) + } + } + }() + + // Function to write data into StashStore and calculate commP + writeFunc := func(f *os.File) error { + limitedReader := io.LimitReader(data, int64(rawSize+1)) // +1 to detect exceeding the limit + + n, err := io.Copy(f, limitedReader) + if err != nil { + return fmt.Errorf("failed to read and write piece data: %w", err) + } + + if n > int64(deal.Data.Size) { + return fmt.Errorf("piece data exceeds the maximum allowed size") + } + + if int64(rawSize) != n { + return fmt.Errorf("raw size does not match with uploaded data: %w", err) + } + + return nil + } + + // Upload into StashStore + stashID, err := m.stor.StashCreate(ctx, int64(deal.Data.Size), writeFunc) + if err != nil { + if err.Error() == "piece data exceeds the maximum allowed size" { + http.Error(w, err.Error(), http.StatusRequestEntityTooLarge) + return + } else if err.Error() == "raw size does not match with uploaded data" { + http.Error(w, err.Error(), http.StatusRequestEntityTooLarge) + return + } else { + log.Errorw("Failed to store piece data in StashStore", "error", err) + http.Error(w, "Failed to store piece data", http.StatusInternalServerError) + return + } + } + + comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + + // 1. Create a long-term parked piece entry + var parkedPieceID int64 + err := tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, TRUE) RETURNING id + `, deal.Data.PieceCID.String(), deal.Data.Size, rawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } + + // 2. Create a piece ref with data_url being "stashstore://" + // Get StashURL + stashURL, err := m.stor.StashURL(stashID) + if err != nil { + return false, fmt.Errorf("failed to get stash URL: %w", err) + } + + stashURL.Scheme = dealdata.CustoreScheme + dataURL := stashURL.String() + + var pieceRefID int64 + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, dataURL).Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, + id.String(), deal.Data.PieceCID.String(), deal.Data.Size, []int64{pieceRefID}) + if err != nil { + return false, xerrors.Errorf("inserting mk20 download pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 download pipeline: %d rows affected", n) + } + + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return false, fmt.Errorf("getting provider ID: %w", err) + } + + ddo := deal.Products.DDOV1 + dealdata := deal.Data + dealID := deal.Identifier.String() + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + var aggregation interface{} + if dealdata.Format.Aggregate != nil { + aggregation = dealdata.Format.Aggregate.Type + } else { + aggregation = nil + } + + n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, + piece_size, raw_size, offline, indexing, announce, + allocation_id, duration, piece_aggregation, started) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE)`, + dealID, spid, ddo.ContractAddress, ddo.Client.String(), dealdata.PieceCID.String(), + dealdata.Size, int64(dealdata.SourceHTTP.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, + allocationID, ddo.Duration, aggregation) + if err != nil { + return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) + } + + return true, nil // Commit the transaction + }, harmonydb.OptionRetry()) + + if err != nil || !comm { + // Remove the stash file as the transaction failed + _ = m.stor.StashRemove(ctx, stashID) + http.Error(w, "Failed to process piece upload", http.StatusInternalServerError) + return + } + + failed = false + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("OK")) +} diff --git a/market/mk20/types.go b/market/mk20/types.go index 5871dc02e..1a0e22166 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -5,6 +5,7 @@ import ( "github.com/ipfs/go-cid" "github.com/oklog/ulid" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" ) @@ -46,7 +47,10 @@ type DataSource struct { // SourceOffline defines the data source for offline pieces, including raw size information. SourceOffline *DataSourceOffline `json:"sourceoffline"` - // SourceHTTPPush // allow clients to push piece data after deal accepted, sort of like offline import + + // SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import + SourceHttpPut *DataSourceHttpPut `json:"sourcehttpput"` + // SourceStorageProvider -> sp IDs/ipni, pieceCids } @@ -87,11 +91,43 @@ type DataSourceOffline struct { RawSize uint64 `json:"rawsize"` } +func (dso *DataSourceOffline) Name() DataSourceName { + return DataSourceNameOffline +} + +func (dso *DataSourceOffline) IsEnabled(dbDataSources []dbDataSource) (int, error) { + name := string(dso.Name()) + for _, p := range dbDataSources { + if p.Name == name { + if p.Enabled { + return Ok, nil + } + } + } + return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) +} + // DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. type DataSourceAggregate struct { Pieces []DataSource `json:"pieces"` } +func (dsa *DataSourceAggregate) Name() DataSourceName { + return DataSourceNameAggregate +} + +func (dsa *DataSourceAggregate) IsEnabled(dbDataSources []dbDataSource) (int, error) { + name := string(dsa.Name()) + for _, p := range dbDataSources { + if p.Name == name { + if p.Enabled { + return Ok, nil + } + } + } + return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) +} + // DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. type DataSourceHTTP struct { @@ -102,6 +138,22 @@ type DataSourceHTTP struct { URLs []HttpUrl `json:"urls"` } +func (dsh *DataSourceHTTP) Name() DataSourceName { + return DataSourceNameHTTP +} + +func (dsh *DataSourceHTTP) IsEnabled(dbDataSources []dbDataSource) (int, error) { + name := string(dsh.Name()) + for _, p := range dbDataSources { + if p.Name == name { + if p.Enabled { + return Ok, nil + } + } + } + return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) +} + // HttpUrl represents an HTTP endpoint configuration for fetching piece data. type HttpUrl struct { @@ -118,6 +170,26 @@ type HttpUrl struct { Fallback bool `json:"fallback"` } +type DataSourceHttpPut struct { + RawSize uint64 `json:"rawsize"` +} + +func (dsh *DataSourceHttpPut) Name() DataSourceName { + return DataSourceNamePut +} + +func (dsh *DataSourceHttpPut) IsEnabled(dbDataSources []dbDataSource) (int, error) { + name := string(dsh.Name()) + for _, p := range dbDataSources { + if p.Name == name { + if p.Enabled { + return Ok, nil + } + } + } + return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) +} + // AggregateType represents an unsigned integer used to define the type of aggregation for data pieces in the system. type AggregateType uint64 @@ -129,26 +201,44 @@ const ( type ErrCode int const ( - Ok = 200 - ErrBadProposal = 400 - ErrMalformedDataSource = 400 - ErrUnsupportedDataSource = 422 - ErrUnsupportedProduct = 422 - ErrProductNotEnabled = 403 - ErrProductValidationRejected = 409 - ErrDealRejectedByMarket = 422 - ErrServiceMaintenance = 503 - ErrServiceOverloaded = 429 - ErrMarketNotEnabled = 440 - ErrDurationTooShort = 441 + Ok = 200 + ErrBadProposal = 400 + ErrMalformedDataSource = 430 + ErrUnsupportedDataSource = 422 + ErrUnsupportedProduct = 423 + ErrProductNotEnabled = 424 + ErrProductValidationFailed = 425 + ErrDealRejectedByMarket = 426 + ErrServiceMaintenance = 503 + ErrServiceOverloaded = 429 + ErrMarketNotEnabled = 440 + ErrDurationTooShort = 441 ) -// TODO: Deal Status - HTTP -// TODO: Supported contractS - HTTP +type ProductName string + +const ( + ProductNameDDOV1 ProductName = "ddov1" +) + +type DataSourceName string + +const ( + DataSourceNameHTTP DataSourceName = "http" + DataSourceNameAggregate DataSourceName = "aggregate" + DataSourceNameOffline DataSourceName = "offline" + DataSourceNameStorageProvider DataSourceName = "storageprovider" + DataSourceNamePDP DataSourceName = "pdp" + DataSourceNamePut DataSourceName = "put" +) + +type dbDataSource struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` +} + // TODO: Client facing UI Page for SP // TODO: Contract SP details pathway - sptool? -// TODO: Error codes use -// TODO: /PUT endpoint // TODO: SPID data source // TODO: Test contract // TODO: ACLv1? diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 252e78390..734ece71f 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -6,9 +6,11 @@ import ( "encoding/json" "fmt" "math/bits" + "net/http" "net/url" "time" + "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "golang.org/x/xerrors" @@ -20,30 +22,41 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" ) -func (d *Deal) Validate() (bool, error) { - if d.Products.DDOV1 == nil { - return false, xerrors.Errorf("no products") +func (d *Deal) Validate(db *harmonydb.DB) (int, error) { + code, err := d.Products.Validate(db) + if err != nil { + return code, xerrors.Errorf("products validation failed: %w", err) } - return d.Data.Validate() + return d.Data.Validate(db) } -func (d *DataSource) Validate() (bool, error) { +func (d DataSource) Validate(db *harmonydb.DB) (int, error) { + var dbDataSources []dbDataSource + + err := db.Select(context.Background(), &dbDataSources, `SELECT name, enabled FROM market_mk20_data_source`) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("getting data sources from DB: %w", err) + } + + if len(dbDataSources) == 0 { + return ErrUnsupportedDataSource, xerrors.Errorf("no data sources enabled on the provider") + } if !d.PieceCID.Defined() { - return false, xerrors.Errorf("piece cid is not defined") + return ErrBadProposal, xerrors.Errorf("piece cid is not defined") } if d.Size == 0 { - return false, xerrors.Errorf("piece size is 0") + return ErrBadProposal, xerrors.Errorf("piece size is 0") } - if d.SourceOffline != nil && d.SourceHTTP != nil && d.SourceAggregate != nil { - return false, xerrors.Errorf("multiple sources defined for data source") + if d.SourceOffline != nil && d.SourceHTTP != nil && d.SourceAggregate != nil && d.SourceHttpPut != nil { + return ErrBadProposal, xerrors.Errorf("multiple sources defined for data source") } - if d.SourceOffline == nil && d.SourceHTTP == nil && d.SourceAggregate == nil { - return false, xerrors.Errorf("no source defined for data source") + if d.SourceOffline == nil && d.SourceHTTP == nil && d.SourceAggregate == nil && d.SourceHttpPut == nil { + return ErrBadProposal, xerrors.Errorf("no source defined for data source") } var fcar, fagg, fraw bool @@ -51,7 +64,7 @@ func (d *DataSource) Validate() (bool, error) { if d.Format.Car != nil { fcar = true if d.Format.Car.Version != 1 && d.Format.Car.Version != 2 { - return false, xerrors.Errorf("car version not supported") + return ErrMalformedDataSource, xerrors.Errorf("car version not supported") } } @@ -59,21 +72,26 @@ func (d *DataSource) Validate() (bool, error) { fagg = true if d.Format.Aggregate.Type != AggregateTypeV1 { - return false, xerrors.Errorf("aggregate type not supported") + return ErrMalformedDataSource, xerrors.Errorf("aggregate type not supported") } if d.SourceAggregate != nil { + code, err := d.SourceAggregate.IsEnabled(dbDataSources) + if err != nil { + return code, err + } + if len(d.SourceAggregate.Pieces) == 0 { - return false, xerrors.Errorf("no pieces in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("no pieces in aggregate") } for _, p := range d.SourceAggregate.Pieces { if !p.PieceCID.Defined() { - return false, xerrors.Errorf("piece cid is not defined") + return ErrMalformedDataSource, xerrors.Errorf("piece cid is not defined") } if p.Size == 0 { - return false, xerrors.Errorf("piece size is 0") + return ErrMalformedDataSource, xerrors.Errorf("piece size is 0") } var ifcar, ifraw bool @@ -81,12 +99,12 @@ func (d *DataSource) Validate() (bool, error) { if p.Format.Car != nil { ifcar = true if p.Format.Car.Version != 1 && p.Format.Car.Version != 2 { - return false, xerrors.Errorf("car version not supported") + return ErrMalformedDataSource, xerrors.Errorf("car version not supported") } } if p.Format.Aggregate != nil { - return false, xerrors.Errorf("aggregate of aggregate is not supported") + return ErrMalformedDataSource, xerrors.Errorf("aggregate of aggregate is not supported") } if p.Format.Raw != nil { @@ -94,45 +112,45 @@ func (d *DataSource) Validate() (bool, error) { } if !ifcar && !ifraw { - return false, xerrors.Errorf("no format defined for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("no format defined for sub piece in aggregate") } if ifcar && ifraw { - return false, xerrors.Errorf("multiple formats defined for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("multiple formats defined for sub piece in aggregate") } if p.SourceAggregate != nil { - return false, xerrors.Errorf("aggregate of aggregate is not supported") + return ErrMalformedDataSource, xerrors.Errorf("aggregate of aggregate is not supported") } if p.SourceOffline == nil && p.SourceHTTP == nil { - return false, xerrors.Errorf("no source defined for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("no source defined for sub piece in aggregate") } if p.SourceOffline != nil && p.SourceHTTP != nil { - return false, xerrors.Errorf("multiple sources defined for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("multiple sources defined for sub piece in aggregate") } if p.SourceHTTP != nil { if p.SourceHTTP.RawSize == 0 { - return false, xerrors.Errorf("raw size is 0 for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("raw size is 0 for sub piece in aggregate") } if len(p.SourceHTTP.URLs) == 0 { - return false, xerrors.Errorf("no urls defined for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("no urls defined for sub piece in aggregate") } for _, u := range d.SourceHTTP.URLs { _, err := url.Parse(u.URL) if err != nil { - return false, xerrors.Errorf("invalid url") + return ErrMalformedDataSource, xerrors.Errorf("invalid url") } } } if p.SourceOffline != nil { if p.SourceOffline.RawSize == 0 { - return false, xerrors.Errorf("raw size is 0 for sub piece in aggregate") + return ErrMalformedDataSource, xerrors.Errorf("raw size is 0 for sub piece in aggregate") } } @@ -145,49 +163,69 @@ func (d *DataSource) Validate() (bool, error) { } if !fcar && !fagg && !fraw { - return false, xerrors.Errorf("no format defined") + return ErrBadProposal, xerrors.Errorf("no format defined") } if fcar && fagg || fcar && fraw || fagg && fraw { - return false, xerrors.Errorf("multiple formats defined") + return ErrBadProposal, xerrors.Errorf("multiple formats defined") } if d.SourceHTTP != nil { + code, err := d.SourceHTTP.IsEnabled(dbDataSources) + if err != nil { + return code, err + } + if d.SourceHTTP.RawSize == 0 { - return false, xerrors.Errorf("raw size is 0") + return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") } if len(d.SourceHTTP.URLs) == 0 { - return false, xerrors.Errorf("no urls defined") + return ErrMalformedDataSource, xerrors.Errorf("no urls defined") } for _, u := range d.SourceHTTP.URLs { _, err := url.Parse(u.URL) if err != nil { - return false, xerrors.Errorf("invalid url") + return ErrMalformedDataSource, xerrors.Errorf("invalid url") } } } if d.SourceOffline != nil { + code, err := d.SourceOffline.IsEnabled(dbDataSources) + if err != nil { + return code, err + } + if d.SourceOffline.RawSize == 0 { - return false, xerrors.Errorf("raw size is 0") + return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") + } + } + + if d.SourceHttpPut != nil { + code, err := d.SourceHttpPut.IsEnabled(dbDataSources) + if err != nil { + return code, err + } + if d.SourceHttpPut.RawSize == 0 { + return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") } } raw, err := d.RawSize() if err != nil { - return false, err + return ErrBadProposal, err } if padreader.PaddedSize(raw).Padded() != d.Size { - return false, xerrors.Errorf("invalid size") + return ErrBadProposal, xerrors.Errorf("invalid size") } - return true, nil + return Ok, nil } -func (d *DataSource) RawSize() (uint64, error) { +func (d DataSource) RawSize() (uint64, error) { if d.Format.Aggregate != nil { if d.Format.Aggregate.Type == AggregateTypeV1 { if d.SourceAggregate != nil { @@ -224,9 +262,38 @@ func (d *DataSource) RawSize() (uint64, error) { if d.SourceOffline != nil { return d.SourceOffline.RawSize, nil } + + if d.SourceHttpPut != nil { + return d.SourceHttpPut.RawSize, nil + } + return 0, xerrors.Errorf("no source defined") } +type dbProduct struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` +} + +func (d Products) Validate(db *harmonydb.DB) (int, error) { + var dbProducts []dbProduct + + err := db.Select(context.Background(), &dbProducts, `SELECT name, enabled FROM products`) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("getting products from DB: %w", err) + } + + if len(dbProducts) == 0 { + return ErrProductNotEnabled, xerrors.Errorf("no products enabled on the provider") + } + + if d.DDOV1 == nil { + return ErrBadProposal, xerrors.Errorf("no products") + } + + return d.DDOV1.Validate(dbProducts) +} + type DBDeal struct { Identifier string `db:"id"` PieceCID string `db:"piece_cid"` @@ -430,17 +497,31 @@ type ProviderDealRejectionInfo struct { } type DealStatusRequest struct { - Identifier string `json:"identifier"` - IdentifierType uint64 `json:"identifiertype"` - ContractAddress string `json:"contractaddress"` + Identifier ulid.ULID `json:"identifier"` + Signature crypto.Signature `json:"signature"` } type DealStatusResponse struct { - Complete bool `json:"complete"` - Error bool `json:"error"` - ErrorMsg string `json:"errormsg"` + State DealState `json:"status"` + ErrorMsg string `json:"errormsg"` +} + +type DealStatus struct { + Response *DealStatusResponse + HTTPCode int } +type DealState string + +const ( + DealStateAccepted DealState = "accepted" + DealStateProcessing DealState = "processing" + DealStateSealing DealState = "sealing" + DealStateIndexing DealState = "indexing" + DealStateFailed DealState = "failed" + DealStateComplete DealState = "complete" +) + type SupportedContracts struct { Contracts []string `json:"contracts"` } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 55b0812e5..0c929caaa 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -101,7 +101,7 @@ func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { func (d *CurioStorageDealMarket) insertDDODealInPipeline(ctx context.Context) { var deals []string - rows, err := d.db.Query(ctx, `SELECT id from market_mk20_pipeline_waiting`) + rows, err := d.db.Query(ctx, `SELECT id from market_mk20_pipeline_waiting WHERE waiting_for_data = FALSE`) if err != nil { log.Errorf("querying mk20 pipeline waiting: %s", err) return diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index ff102e898..373417f43 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -37,7 +37,7 @@ import ( "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/market/storageingest" - "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/proofs" "github.com/filecoin-project/lotus/storage/pipeline/piece" ) @@ -78,6 +78,7 @@ type CurioStorageDealMarket struct { urls map[string]http.Header adders [numPollers]promise.Promise[harmonytask.AddTaskFunc] as *multictladdr.MultiAddressSelector + stor paths.StashStore } type MK12Pipeline struct { @@ -114,7 +115,7 @@ type MK12Pipeline struct { Offset *int64 `db:"sector_offset"` } -func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector) *CurioStorageDealMarket { +func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector, stor paths.StashStore) *CurioStorageDealMarket { moduleMap := make(map[string][]address.Address) moduleMap[mk12Str] = append(moduleMap[mk12Str], miners...) @@ -133,6 +134,7 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * urls: urls, as: as, ethClient: ethClient, + stor: stor, } } @@ -183,7 +185,7 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { if len(miners) == 0 { return nil } - d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as) + d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as, d.stor) if err != nil { return err } @@ -719,10 +721,10 @@ func (d *CurioStorageDealMarket) ingestDeal(ctx context.Context, deal MK12Pipeli StartEpoch: abi.ChainEpoch(dbdeal.StartEpoch), EndEpoch: abi.ChainEpoch(dbdeal.EndEpoch), }, - PieceActivationManifest: &miner.PieceActivationManifest{ + PieceActivationManifest: &lminer.PieceActivationManifest{ CID: pcid, Size: abi.PaddedPieceSize(dbdeal.PieceSize), - VerifiedAllocationKey: &miner.VerifiedAllocationKey{ + VerifiedAllocationKey: &lminer.VerifiedAllocationKey{ Client: abi.ActorID(clientId), ID: verifreg.AllocationId(dbdeal.AllocationID), }, From 7f47865025df91c5d43f68dce515a2e3abcafade Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 14 May 2025 18:59:35 +0400 Subject: [PATCH 03/55] add some type validation tests --- market/mk20/mk20.go | 25 ++- market/mk20/types_test.go | 330 ++++++++++++++++++++++++++++++++++++++ market/mk20/utils.go | 32 ++-- 3 files changed, 365 insertions(+), 22 deletions(-) create mode 100644 market/mk20/types_test.go diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 410d63b7a..fcda9f5d1 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -75,7 +75,30 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { // Validate the DataSource TODO: Add error code to validate - code, err := deal.Validate(m.db) + var dbProducts []dbProduct + err := m.db.Select(context.Background(), &dbProducts, `SELECT name, enabled FROM products`) + if err != nil { + log.Errorw("error getting products from DB", "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + + var dbDataSources []dbDataSource + err = m.db.Select(context.Background(), &dbDataSources, `SELECT name, enabled FROM data_sources`) + if err != nil { + log.Errorw("error getting data sources from DB", "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + + vdata := &productAndDataSource{ + Products: dbProducts, + Data: dbDataSources, + } + + code, err := deal.Validate(vdata) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go new file mode 100644 index 000000000..3d434e110 --- /dev/null +++ b/market/mk20/types_test.go @@ -0,0 +1,330 @@ +package mk20 + +import ( + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" +) + +var ( + testCID, _ = cid.Parse("baga6ea4seaqnnlsm75qhc4h76ts6bytfdxf6epjgqlhozjtuony4fwlui2xfuhq") + raw2MiB = uint64(2 << 20) // unā€‘padded 2MiB + padded2MiB = padreader.PaddedSize(raw2MiB).Padded() +) + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// helpers to create *valid* structs that individual testā€‘cases mutate +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +func validDBDataSources() []dbDataSource { + return []dbDataSource{ + {Name: "http", Enabled: true}, + {Name: "offline", Enabled: true}, + {Name: "aggregate", Enabled: true}, + {Name: "put", Enabled: true}, + } +} + +func validDBProducts() []dbProduct { return []dbProduct{{Name: "ddov1", Enabled: true}} } + +func validDataSource() DataSource { + return DataSource{ + PieceCID: testCID, + Size: padded2MiB, + Format: PieceDataFormat{ + Car: &FormatCar{Version: 1}, + }, + SourceHTTP: &DataSourceHTTP{ + RawSize: raw2MiB, + URLs: []HttpUrl{{URL: "https://example.com/file.car"}}, + }, + } +} + +func validDDOV1() *DDOV1 { + sp, _ := address.NewFromString("f01234") + cl, _ := address.NewFromString("f05678") + pm, _ := address.NewFromString("f09999") + + return &DDOV1{ + Provider: sp, + Client: cl, + PieceManager: pm, + Duration: 518400, + ContractAddress: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ContractDealIDMethod: "dealID", + ContractDealIDMethodParams: []byte{0x01}, + } +} + +func validDeal(t *testing.T) Deal { + id, err := NewULID() + require.NoError(t, err) + return Deal{ + Identifier: id, + Data: validDataSource(), + Products: Products{DDOV1: validDDOV1()}, + } +} + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// 1. Products.Validate + DDOV1.Validate +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +func TestValidate_DDOV1(t *testing.T) { + base := *validDDOV1() // copy + tests := []struct { + name string + prod []dbProduct + mutate func(*DDOV1) + wantCode int + }{ + // enabled / disabled / unsupported + {"no products on provider", + nil, + func(d *DDOV1) {}, + ErrUnsupportedProduct}, + {"product disabled", + []dbProduct{{Name: "ddov1", Enabled: false}}, + func(d *DDOV1) {}, + ErrProductNotEnabled}, + {"product unsupported", + []dbProduct{{Name: "other", Enabled: true}}, + func(d *DDOV1) {}, + ErrUnsupportedProduct}, + + // fieldā€‘level failures + {"provider undef", validDBProducts(), + func(d *DDOV1) { d.Provider = address.Undef }, + ErrProductValidationFailed}, + {"client undef", validDBProducts(), + func(d *DDOV1) { d.Client = address.Undef }, + ErrProductValidationFailed}, + {"pieceā€‘manager undef", validDBProducts(), + func(d *DDOV1) { d.PieceManager = address.Undef }, + ErrProductValidationFailed}, + {"allocation id == NoAllocationID", validDBProducts(), + func(d *DDOV1) { + na := verifreg.NoAllocationID + d.AllocationId = &na + }, + ErrProductValidationFailed}, + {"duration too short", validDBProducts(), + func(d *DDOV1) { d.Duration = 10 }, + ErrDurationTooShort}, + {"contract address empty", validDBProducts(), + func(d *DDOV1) { d.ContractAddress = "" }, + ErrProductValidationFailed}, + {"contract address no 0x", validDBProducts(), + func(d *DDOV1) { d.ContractAddress = "abc" }, + ErrProductValidationFailed}, + {"contract params nil", validDBProducts(), + func(d *DDOV1) { d.ContractDealIDMethodParams = nil }, + ErrProductValidationFailed}, + {"contract method empty", validDBProducts(), + func(d *DDOV1) { d.ContractDealIDMethod = "" }, + ErrProductValidationFailed}, + + // happy path + {"happy path", validDBProducts(), + func(d *DDOV1) {}, + Ok}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + d := base + tc.mutate(&d) + code, _ := d.Validate(tc.prod) + require.Equal(t, tc.wantCode, code) + }) + } +} + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// 2. DataSource.Validate (all branches) +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +func TestValidate_DataSource(t *testing.T) { + baseDB := validDBDataSources() + tests := []struct { + name string + mutateDS func(*DataSource) + mutateDBSrc func([]dbDataSource) []dbDataSource + wantCode int + }{ + // providerā€‘level enable / disable checks + {"no data sources enabled", + func(ds *DataSource) {}, func(_ []dbDataSource) []dbDataSource { return nil }, + ErrUnsupportedDataSource}, + {"http disabled", + func(ds *DataSource) {}, + func(src []dbDataSource) []dbDataSource { src[0].Enabled = false; return src }, + ErrUnsupportedDataSource}, + + // topā€‘level sanity + {"undefined CID", + func(ds *DataSource) { ds.PieceCID = cid.Undef }, + nil, ErrBadProposal}, + {"size zero", + func(ds *DataSource) { ds.Size = 0 }, + nil, ErrBadProposal}, + {"no source defined", + func(ds *DataSource) { + ds.SourceHTTP = nil + }, nil, ErrBadProposal}, + {"multiple sources defined", + func(ds *DataSource) { + ds.SourceOffline = &DataSourceOffline{RawSize: raw2MiB} + ds.SourceHttpPut = &DataSourceHttpPut{RawSize: raw2MiB} + ds.SourceAggregate = &DataSourceAggregate{Pieces: []DataSource{}} + }, nil, ErrBadProposal}, + + // format combinations + {"no format", + func(ds *DataSource) { ds.Format = PieceDataFormat{} }, + nil, ErrBadProposal}, + {"multiple formats", + func(ds *DataSource) { + ds.Format.Raw = &FormatBytes{} + }, nil, ErrBadProposal}, + {"car version unsupported", + func(ds *DataSource) { ds.Format.Car.Version = 3 }, + nil, ErrMalformedDataSource}, + + // HTTP source specific + {"http rawsize zero", + func(ds *DataSource) { ds.SourceHTTP.RawSize = 0 }, + nil, ErrMalformedDataSource}, + {"http urls empty", + func(ds *DataSource) { ds.SourceHTTP.URLs = nil }, + nil, ErrMalformedDataSource}, + {"http url invalid", + func(ds *DataSource) { ds.SourceHTTP.URLs[0].URL = "::::" }, + nil, ErrMalformedDataSource}, + + // Offline source + {"offline source disabled", + func(ds *DataSource) { + ds.SourceHTTP = nil + ds.SourceOffline = &DataSourceOffline{RawSize: raw2MiB} + }, + func(src []dbDataSource) []dbDataSource { src[1].Enabled = false; return src }, + ErrUnsupportedDataSource}, + {"offline rawsize zero", + func(ds *DataSource) { + ds.SourceHTTP = nil + ds.SourceOffline = &DataSourceOffline{RawSize: 0} + }, nil, ErrMalformedDataSource}, + + // HttpPut source + {"put source disabled", + func(ds *DataSource) { + ds.SourceHTTP = nil + ds.SourceHttpPut = &DataSourceHttpPut{RawSize: raw2MiB} + }, + func(src []dbDataSource) []dbDataSource { src[3].Enabled = false; return src }, + ErrUnsupportedDataSource}, + {"put rawsize zero", + func(ds *DataSource) { + ds.SourceHTTP = nil + ds.SourceHttpPut = &DataSourceHttpPut{RawSize: 0} + }, nil, ErrMalformedDataSource}, + + // Size mismatch on final check + {"declared size mismatch", + func(ds *DataSource) { ds.Size *= 2 }, + nil, ErrBadProposal}, + + // happy path + {"happy path", + func(ds *DataSource) {}, + nil, Ok}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + ds := validDataSource() + tc.mutateDS(&ds) + + db := baseDB + if tc.mutateDBSrc != nil { + db = tc.mutateDBSrc(append([]dbDataSource(nil), baseDB...)) + } + code, _ := ds.Validate(db) + require.Equal(t, tc.wantCode, code) + }) + } +} + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// 3. Deal.Validate (composition of the two) +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +func TestValidate_Deal(t *testing.T) { + tests := []struct { + name string + mutate func(*Deal, *productAndDataSource) + wantCode int + }{ + {"happy path", + func(d *Deal, _ *productAndDataSource) {}, + Ok}, + + // propagate product failure + {"product failure bubbles", + func(d *Deal, pad *productAndDataSource) { + pad.Products[0].Enabled = false // DDOV1 disabled + }, ErrProductNotEnabled}, + + // propagate data source failure + {"data source failure bubbles", + func(d *Deal, _ *productAndDataSource) { + d.Data.PieceCID = cid.Undef + }, ErrBadProposal}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + deal := validDeal(t) + pad := &productAndDataSource{ + Products: append([]dbProduct(nil), validDBProducts()...), + Data: append([]dbDataSource(nil), validDBDataSources()...), + } + tc.mutate(&deal, pad) + code, _ := deal.Validate(pad) + require.Equal(t, tc.wantCode, code) + }) + } +} + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// 4. quick sanity that URL parsing in Validate works on aggregate subā€‘pieces +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +func TestValidate_Aggregate_SubPieceChecks(t *testing.T) { + // base structure: an aggregate of one valid HTTP piece + sub := validDataSource() + agg := validDataSource() + agg.Format = PieceDataFormat{ + Aggregate: &FormatAggregate{ + Type: AggregateTypeV1, + Sub: nil, + }, + } + agg.SourceHTTP = nil + agg.SourceAggregate = &DataSourceAggregate{Pieces: []DataSource{sub}} + + // (size will mismatch ā€“ test expects that specific error branch) + agg.Size = padded2MiB * 8 + + code, _ := agg.Validate(validDBDataSources()) + require.Equal(t, ErrBadProposal, code) +} diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 734ece71f..39750704c 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "math/bits" - "net/http" "net/url" "time" @@ -22,23 +21,21 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" ) -func (d *Deal) Validate(db *harmonydb.DB) (int, error) { - code, err := d.Products.Validate(db) +type productAndDataSource struct { + Products []dbProduct + Data []dbDataSource +} + +func (d *Deal) Validate(pad *productAndDataSource) (int, error) { + code, err := d.Products.Validate(pad.Products) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) } - return d.Data.Validate(db) + return d.Data.Validate(pad.Data) } -func (d DataSource) Validate(db *harmonydb.DB) (int, error) { - var dbDataSources []dbDataSource - - err := db.Select(context.Background(), &dbDataSources, `SELECT name, enabled FROM market_mk20_data_source`) - if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("getting data sources from DB: %w", err) - } - +func (d DataSource) Validate(dbDataSources []dbDataSource) (int, error) { if len(dbDataSources) == 0 { return ErrUnsupportedDataSource, xerrors.Errorf("no data sources enabled on the provider") } @@ -140,7 +137,7 @@ func (d DataSource) Validate(db *harmonydb.DB) (int, error) { return ErrMalformedDataSource, xerrors.Errorf("no urls defined for sub piece in aggregate") } - for _, u := range d.SourceHTTP.URLs { + for _, u := range p.SourceHTTP.URLs { _, err := url.Parse(u.URL) if err != nil { return ErrMalformedDataSource, xerrors.Errorf("invalid url") @@ -275,14 +272,7 @@ type dbProduct struct { Enabled bool `db:"enabled"` } -func (d Products) Validate(db *harmonydb.DB) (int, error) { - var dbProducts []dbProduct - - err := db.Select(context.Background(), &dbProducts, `SELECT name, enabled FROM products`) - if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("getting products from DB: %w", err) - } - +func (d Products) Validate(dbProducts []dbProduct) (int, error) { if len(dbProducts) == 0 { return ErrProductNotEnabled, xerrors.Errorf("no products enabled on the provider") } From 7ca62764620e3ca26dc5846506fe7c6ea11c03df Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 16 May 2025 21:29:31 +0400 Subject: [PATCH 04/55] info page and pdp router changes --- Makefile | 6 +- cuhttp/server.go | 11 +- deps/config/types.go | 2 +- .../harmonydb/sql/20250505-market_mk20.sql | 2 +- market/http/http.go | 26 +- market/mk20/ddo_v1.go | 6 +- market/mk20/http/http.go | 38 +- market/mk20/info.md | 255 +++++++++++++ market/mk20/mk20.go | 4 +- market/mk20/mk20_utils.go | 15 +- market/mk20/mk20gen/gen.go | 361 ++++++++++++++++++ market/mk20/types.go | 81 ++-- market/mk20/types_test.go | 11 +- market/mk20/utils.go | 56 ++- pdp/handlers.go | 19 +- pdp/handlers_upload.go | 2 +- 16 files changed, 805 insertions(+), 90 deletions(-) create mode 100644 market/mk20/info.md create mode 100644 market/mk20/mk20gen/gen.go diff --git a/Makefile b/Makefile index 4003a7feb..01f4260ea 100644 --- a/Makefile +++ b/Makefile @@ -266,7 +266,11 @@ go-generate: gen: gensimple .PHONY: gen -gensimple: api-gen go-generate cfgdoc-gen docsgen docsgen-cli +marketgen: + $(GOCC) run ./market/mk20/mk20gen -pkg ./market/mk20 -output ./market/mk20/info.md +.PHONY: marketgen + +gensimple: api-gen go-generate cfgdoc-gen docsgen marketgen docsgen-cli $(GOCC) run ./scripts/fiximports go mod tidy .PHONY: gen diff --git a/cuhttp/server.go b/cuhttp/server.go index 716da25a7..8d264fd0b 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -24,7 +24,6 @@ import ( ipni_provider "github.com/filecoin-project/curio/market/ipni/ipni-provider" "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/retrieval" - "github.com/filecoin-project/curio/pdp" "github.com/filecoin-project/curio/tasks/message" storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) @@ -296,13 +295,13 @@ func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDep rd := libp2p.NewRedirector(d.DB) libp2p.Router(r, rd) - if sd.EthSender != nil { - pdsvc := pdp.NewPDPService(d.DB, d.LocalStore, must.One(d.EthClient.Get()), d.Chain, sd.EthSender) - pdp.Routes(r, pdsvc) - } + //if sd.EthSender != nil { + // pdsvc := pdp.NewPDPService(d.DB, d.LocalStore, must.One(d.EthClient.Get()), d.Chain, sd.EthSender) + // pdp.Routes(r, pdsvc) + //} // Attach the market handler - dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket) + dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket, must.One(d.EthClient.Get()), d.Chain, sd.EthSender, d.LocalStore) if err != nil { return nil, xerrors.Errorf("failed to create new market handler: %w", err) } diff --git a/deps/config/types.go b/deps/config/types.go index a7ab451d5..35eb68f86 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -124,7 +124,7 @@ func DefaultCurioConfig() *CurioConfig { DomainName: "", ListenAddress: "0.0.0.0:12310", ReadTimeout: time.Second * 10, - IdleTimeout: time.Minute * 2, + IdleTimeout: time.Hour, ReadHeaderTimeout: time.Second * 5, EnableCORS: true, CSP: "inline", diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index f9881d7d1..90d07f2cc 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -68,7 +68,7 @@ CREATE TABLE market_mk20_pipeline_waiting ( id TEXT PRIMARY KEY, waiting_for_data BOOLEAN DEFAULT FALSE, started_put BOOLEAN DEFAULT FALSE, - start_time TIMESTAMPZ DEFAULT NULL + start_time TIMESTAMPTZ DEFAULT NULL ); CREATE TABLE market_mk20_download_pipeline ( diff --git a/market/http/http.go b/market/http/http.go index b4fb23803..79001d0e5 100644 --- a/market/http/http.go +++ b/market/http/http.go @@ -1,23 +1,28 @@ package http import ( + "github.com/ethereum/go-ethereum/ethclient" "github.com/go-chi/chi/v5" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/paths" mk12http "github.com/filecoin-project/curio/market/mk12/http" mk20http "github.com/filecoin-project/curio/market/mk20/http" + "github.com/filecoin-project/curio/pdp" + "github.com/filecoin-project/curio/tasks/message" storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) type MarketHandler struct { - mdh12 *mk12http.MK12DealHandler - mdh20 *mk20http.MK20DealHandler + mdh12 *mk12http.MK12DealHandler + mdh20 *mk20http.MK20DealHandler + pdpService *pdp.PDPService } // NewMarketHandler is used to prepare all the required market handlers. Currently, it supports mk12 deal market. // This function should be used to expand the functionality under "/market" path -func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_market.CurioStorageDealMarket) (*MarketHandler, error) { +func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_market.CurioStorageDealMarket, eth *ethclient.Client, fc pdp.PDPServiceNodeApi, sn *message.SenderETH, stor paths.StashStore) (*MarketHandler, error) { mdh12, err := mk12http.NewMK12DealHandler(db, cfg, dm) if err != nil { return nil, err @@ -28,9 +33,17 @@ func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_mar return nil, err } + var pdpService *pdp.PDPService + + if sn != nil { + pdpService = pdp.NewPDPService(db, stor, eth, fc, sn) + //pdp.Routes(r, pdsvc) + } + return &MarketHandler{ - mdh12: mdh12, - mdh20: mdh20, + mdh12: mdh12, + mdh20: mdh20, + pdpService: pdpService, }, nil } @@ -39,5 +52,8 @@ func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_mar func Router(mux *chi.Mux, mh *MarketHandler) { mux.Mount("/market/mk12", mk12http.Router(mh.mdh12)) mux.Mount("/market/mk20", mk20http.Router(mh.mdh20)) + if mh.pdpService != nil { + mux.Mount("/market/pdp", pdp.Routes(mh.pdpService)) + } // TODO: Attach a info endpoint here with details about supported market modules and services under them } diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index 62026ec1c..9de4fe2bb 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -65,7 +65,7 @@ type DDOV1 struct { AnnounceToIPNI bool `json:"announcetoinpni"` } -func (d *DDOV1) Validate(dbProducts []dbProduct) (int, error) { +func (d *DDOV1) Validate(dbProducts []dbProduct) (ErrorCode, error) { code, err := d.IsEnabled(dbProducts) if err != nil { return code, err @@ -114,7 +114,7 @@ func (d *DDOV1) Validate(dbProducts []dbProduct) (int, error) { return Ok, nil } -func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, int, error) { +func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, ErrorCode, error) { var abiStr string err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) if err != nil { @@ -177,7 +177,7 @@ func (d *DDOV1) ProductName() ProductName { return ProductNameDDOV1 } -func (d *DDOV1) IsEnabled(dbProducts []dbProduct) (int, error) { +func (d *DDOV1) IsEnabled(dbProducts []dbProduct) (ErrorCode, error) { name := string(d.ProductName()) for _, p := range dbProducts { if p.Name == name { diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index e29371577..ea05ddf1c 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "time" @@ -53,14 +54,21 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { func Router(mdh *MK20DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Post("/store", mdh.mk20deal) - //mux.Get("/ask", mdh.mk20ask) - mux.Get("/status", mdh.mk20status) - mux.Get("/contracts", mdh.mk20supportedContracts) + mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), 10*time.Second, "timeout reading request")) + mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), 10*time.Second, "timeout reading request")) + mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), 10*time.Second, "timeout reading request")) mux.Put("/data", mdh.mk20UploadDealData) + mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), 10*time.Second, "timeout reading request")) + //mux.Post("/store", mdh.mk20deal) + //mux.Get("/status", mdh.mk20status) + //mux.Get("/contracts", mdh.mk20supportedContracts) + //mux.Get("/info", mdh.info) return mux } +// mk20deal handles incoming HTTP POST requests to process MK20 deals. +// It validates the request's content type and body, then parses and executes the deal logic. +// Responds with appropriate HTTP status codes and logs detailed information about the process. func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") var deal mk20.Deal @@ -97,6 +105,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { } } +// mk20status handles HTTP requests to retrieve the status of a deal using its ID, responding with deal status or appropriate error codes. func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { // Extract id from the URL idStr := chi.URLParam(r, "id") @@ -133,6 +142,7 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { } } +// mk20supportedContracts retrieves supported contract addresses from the database and returns them as a JSON response. func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *http.Request) { var contracts mk20.SupportedContracts err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM contracts") @@ -156,6 +166,7 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt } } +// mk20UploadDealData handles uploading deal data to the server using a PUT request with specific validations and streams directly to the logic. func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Request) { // Extract id from the URL idStr := chi.URLParam(r, "id") @@ -186,5 +197,22 @@ func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Re } // Stream directly to execution logic - mdh.dm.MK20Handler.HandlePutRequest(context.Background(), id, r.Body, w) + mdh.dm.MK20Handler.HandlePutRequest(id, r.Body, w) +} + +// info serves the contents of the info file as a text/markdown response with HTTP 200 or returns an HTTP 500 on read/write failure. +func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { + // Read the info File + data, err := os.ReadFile("../info.md") + if err != nil { + log.Errorw("failed to read info file", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/markdown") + _, err = w.Write(data) + if err != nil { + log.Errorw("failed to write info file", "err", err) + } } diff --git a/market/mk20/info.md b/market/mk20/info.md new file mode 100644 index 000000000..19c57b3dd --- /dev/null +++ b/market/mk20/info.md @@ -0,0 +1,255 @@ +# Storage Market Interface + +This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider. + +## šŸ“” MK20 HTTP API Overview + +The MK20 storage market module provides a set of HTTP endpoints under `/market/mk20` that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior. + +### Base URL + +The base URL for all MK20 endpoints is: + +``` + +/market/mk20 + +``` + +### šŸ”„ POST /store + +Submit a new MK20 deal. + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A +- **Response**: + - `200 OK`: Deal accepted + - Other [HTTP codes](#constants-for-errorcode) indicate validation failure, rejection, or system errors + +### šŸ§¾ GET /status?id= + +Retrieve the current status of a deal. + +- **Content-Type**: `application/json` +- **Body**: N/A +- **Query Parameters**: + - `id`: Deal identifier in [ULID](https://github.com/ulid/spec) format +- **Response**: + - `200 OK`: JSON-encoded [deal status](#dealstatusresponse) information + - `400 Bad Request`: Missing or invalid ID + - `500 Internal Server Error`: If backend fails to respond + +### šŸ“œ GET /contracts + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A +Return the list of contract addresses supported by the provider. + +- **Response**: + - `200 OK`: [JSON array of contract addresses](#supportedcontracts) + - `500 Internal Server Error`: Query or serialization failure + +### šŸ—‚ PUT /data?id= + +Upload deal data after the deal has been accepted. + +- **Content-Type**: `application/octet-stream` +- **Body**: Deal data bytes +- **Query Parameter**: + -`id`: Deal identifier in [ULID](https://github.com/ulid/spec) format +- **Headers**: + - `Content-Length`: must be deal's raw size +- **Response**: + - `200 OK`: if data is successfully streamed + - `400`, `413`, or `415`: on validation failures + +### šŸ§  GET /info + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A +Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources. + +- **Response**: + - `200 OK`: with markdown content of the info file + - `500 Internal Server Error`: if file is not found or cannot be read + +## Supported Deal Types + +This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission. + +### Deal + +Deal represents a structure defining the details and components of a specific deal in the system. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Identifier | [ulid.ULID](https://pkg.go.dev/github.com/oklog/ulid#ULID) | json:"identifier" | Identifier represents a unique identifier for the deal in UUID format. | +| Data | [mk20.DataSource](#datasource) | json:"data" | Data represents the source of piece data and associated metadata. | +| Products | [mk20.Products](#products) | json:"products" | Products represents a collection of product-specific information associated with a deal | + +### DataSource + +DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piececid" | PieceCID represents the unique identifier for a piece of data, stored as a CID object. | +| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"size" | Size represents the size of the padded piece in the data source. | +| Format | [mk20.PieceDataFormat](#piecedataformat) | json:"format" | Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. | +| SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"sourcehttp" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | +| SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"sourceaggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | +| SourceOffline | [*mk20.DataSourceOffline](#datasourceoffline) | json:"sourceoffline" | SourceOffline defines the data source for offline pieces, including raw size information. | +| SourceHttpPut | [*mk20.DataSourceHttpPut](#datasourcehttpput) | json:"sourcehttpput" | SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import | + +### Products + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| DDOV1 | [*mk20.DDOV1](#ddov1) | json:"ddov1" | DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) | + +### DDOV1 + +DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details +for a DDO deal handling. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Provider | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"provider" | Provider specifies the address of the provider | +| Client | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"client" | Client represents the address of the deal client | +| PieceManager | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"piecemanager" | Actor able to with AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs | +| Duration | [abi.ChainEpoch](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#ChainEpoch) | json:"duration" | Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400 | +| AllocationId | [*verifreg.AllocationId](https://pkg.go.dev/github.com/filecoin-project/go-state-types/builtin/v16/verifreg#AllocationId) | json:"aggregatedallocationid" | AllocationId represents an aggregated allocation identifier for the deal. | +| ContractAddress | [string](https://pkg.go.dev/builtin#string) | json:"contractaddress" | ContractAddress specifies the address of the contract governing the deal | +| ContractDealIDMethod | [string](https://pkg.go.dev/builtin#string) | json:"contractdealidmethod" | ContractDealIDMethod specifies the method name to retrieve the deal ID for a contract | +| ContractDealIDMethodParams | [[]byte](https://pkg.go.dev/builtin#byte) | json:"contractdealidmethodparams" | ContractDealIDMethodParams represents encoded parameters for the contract deal ID method if required by the contract | +| NotificationAddress | [string](https://pkg.go.dev/builtin#string) | json:"notificationaddress" | NotificationAddress specifies the address to which notifications will be relayed to when sector is activated | +| NotificationPayload | [[]byte](https://pkg.go.dev/builtin#byte) | json:"notificationpayload" | NotificationPayload holds the notification data typically in a serialized byte array format. | +| Indexing | [bool](https://pkg.go.dev/builtin#bool) | json:"indexing" | Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval | +| AnnounceToIPNI | [bool](https://pkg.go.dev/builtin#bool) | json:"announcetoinpni" | AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). | + +### DataSourceAggregate + +DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Pieces | [[]mk20.DataSource](#datasource) | json:"pieces" | | + +### DataSourceHTTP + +DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | +| URLs | [[]mk20.HttpUrl](#httpurl) | json:"urls" | URLs lists the HTTP endpoints where the piece data can be fetched. | + +### DataSourceHttpPut + +DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | + +### DataSourceOffline + +DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | + +### DealStatusResponse + +DealStatusResponse represents the response of a deal's status, including its current state and an optional error message. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| State | [mk20.DealState](#constants-for-dealstate) | json:"status" | State indicates the current processing state of the deal as a DealState value. | +| ErrorMsg | [string](https://pkg.go.dev/builtin#string) | json:"errormsg" | ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. | + +### FormatAggregate + +FormatAggregate represents the aggregated format for piece data, identified by its type. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Type | [mk20.AggregateType](https://pkg.go.dev/github.com/filecoin-project/curio/market/mk20#AggregateType) | json:"type" | Type specifies the type of aggregation for data pieces, represented by an AggregateType value. | +| Sub | [[]mk20.PieceDataFormat](#piecedataformat) | json:"sub" | Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate | + +### FormatBytes + +FormatBytes defines the raw byte representation of data as a format. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| + +### FormatCar + +FormatCar represents the CAR (Content Addressable aRchive) format with version metadata for piece data serialization. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Version | [uint64](https://pkg.go.dev/builtin#uint64) | json:"version" | Version specifies the version of the CAR format used for piece data serialization. | + +### HttpUrl + +HttpUrl represents an HTTP endpoint configuration for fetching piece data. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| URL | [string](https://pkg.go.dev/builtin#string) | json:"url" | URL specifies the HTTP endpoint where the piece data can be fetched. | +| HTTPHeaders | [http.Header](https://pkg.go.dev/net/http#Header) | json:"httpheaders" | HTTPHeaders represents the HTTP headers associated with the URL. | +| Priority | [uint64](https://pkg.go.dev/builtin#uint64) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | +| Fallback | [bool](https://pkg.go.dev/builtin#bool) | json:"fallback" | Fallback indicates whether this URL serves as a fallback option when other URLs fail. | + +### PieceDataFormat + +PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Car | [*mk20.FormatCar](#formatcar) | json:"car" | Car represents the optional CAR file format, including its metadata and versioning details. | +| Aggregate | [*mk20.FormatAggregate](#formataggregate) | json:"aggregate" | Aggregate holds a reference to the aggregated format of piece data. | +| Raw | [*mk20.FormatBytes](#formatbytes) | json:"raw" | Raw represents the raw format of the piece data, encapsulated as bytes. | + +### SupportedContracts + +SupportedContracts represents a collection of contract addresses supported by a system or application. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Contracts | [[]string](https://pkg.go.dev/builtin#string) | json:"contracts" | Contracts represents a list of supported contract addresses in string format. | + +### Constants for ErrorCode + +| Constant | Code | Description | +|----------|------|-------------| +| Ok | 200 | Ok represents a successful operation with an HTTP status code of 200. | +| ErrBadProposal | 400 | ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic. | +| ErrMalformedDataSource | 430 | ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data. | +| ErrUnsupportedDataSource | 422 | ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context. | +| ErrUnsupportedProduct | 423 | ErrUnsupportedProduct indicates that the requested product is not supported by the provider. | +| ErrProductNotEnabled | 424 | ErrProductNotEnabled indicates that the requested product is not enabled on the provider. | +| ErrProductValidationFailed | 425 | ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data. | +| ErrDealRejectedByMarket | 426 | ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules. | +| ErrServiceMaintenance | 503 | ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503. | +| ErrServiceOverloaded | 429 | ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment. | +| ErrMarketNotEnabled | 440 | ErrMarketNotEnabled indicates that the market is not enabled for the requested operation. | +| ErrDurationTooShort | 441 | ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold. | + +### Constants for DealState + +| Constant | Code | Description | +|----------|------|-------------| +| DealStateAccepted | "accepted" | DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system. | +| DealStateProcessing | "processing" | DealStateProcessing represents the state of a deal currently being processed in the pipeline. | +| DealStateSealing | "sealing" | DealStateSealing indicates that the deal is currently being sealed in the system. | +| DealStateIndexing | "indexing" | DealStateIndexing represents the state where a deal is undergoing indexing in the system. | +| DealStateFailed | "failed" | DealStateFailed indicates that the deal has failed due to an error during processing, sealing, or indexing. | +| DealStateComplete | "complete" | DealStateComplete indicates that the deal has successfully completed all processing and is finalized in the system. | + diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index fcda9f5d1..7323430a3 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -102,7 +102,7 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ - HTTPCode: code, + HTTPCode: int(code), } if code == http.StatusInternalServerError { ret.Reason = "Internal server error" @@ -130,7 +130,7 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje if err != nil { log.Errorw("error getting deal ID", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ - HTTPCode: code, + HTTPCode: int(code), } if code == http.StatusInternalServerError { ret.Reason = "Internal server error" diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index 618891af9..39e719176 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -10,15 +10,15 @@ import ( "os" "time" - "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/lib/dealdata" - "github.com/filecoin-project/go-address" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" -) -const PutGracePeriod = time.Hour + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/dealdata" +) func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { // Check if we ever accepted this deal @@ -132,7 +132,8 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { } } -func (m *MK20) HandlePutRequest(ctx context.Context, id ulid.ULID, data io.ReadCloser, w http.ResponseWriter) { +func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.ResponseWriter) { + ctx := context.Background() defer data.Close() var waitingDeal []struct { @@ -153,7 +154,7 @@ func (m *MK20) HandlePutRequest(ctx context.Context, id ulid.ULID, data io.ReadC http.Error(w, "", http.StatusNotFound) } - if waitingDeal[0].Started && waitingDeal[0].StartTime.Add(PutGracePeriod).Before(time.Now()) { + if waitingDeal[0].Started && waitingDeal[0].StartTime.Add(m.cfg.HTTP.ReadTimeout).Before(time.Now()) { http.Error(w, "another /PUT request is in progress for this deal", http.StatusConflict) } diff --git a/market/mk20/mk20gen/gen.go b/market/mk20/mk20gen/gen.go new file mode 100644 index 000000000..fce1c85a1 --- /dev/null +++ b/market/mk20/mk20gen/gen.go @@ -0,0 +1,361 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/doc" + "go/token" + "go/types" + "log" + "os" + "sort" + "strings" + + "golang.org/x/tools/go/packages" +) + +type StructInfo struct { + Name string + Doc string + Fields []FieldInfo +} + +type FieldInfo struct { + Name string + Type string + Tag string + Doc string + Typ types.Type // ā† add this field +} + +type constEntry struct { + Name string + Value string + Doc string +} + +var visited = map[string]bool{} +var structMap = map[string]StructInfo{} +var rendered = map[string]bool{} +var constMap = map[string][]constEntry{} + +var skipTypes = map[string]bool{ + "ProviderDealRejectionInfo": true, + "DBDeal": true, + "dbProduct": true, + "dbDataSource": true, + "productAndDataSource": true, + "MK20": true, + "DealStatus": true, +} + +var includeConsts = map[string]bool{ + "ErrorCode": true, + "DealState": true, +} + +func main() { + var pkgPath, output string + flag.StringVar(&pkgPath, "pkg", "./", "Package to scan") + flag.StringVar(&output, "output", "info.md", "Output file") + flag.Parse() + + cfg := &packages.Config{ + Mode: packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports | packages.NeedName | packages.NeedFiles, + Fset: token.NewFileSet(), + } + + pkgs, err := packages.Load(cfg, pkgPath) + if err != nil { + log.Fatalf("Failed to load package: %v", err) + } + + for _, pkg := range pkgs { + docPkg, err := doc.NewFromFiles(cfg.Fset, pkg.Syntax, pkg.PkgPath) + if err != nil { + log.Fatalf("Failed to parse package: %v", err) + } + for _, t := range docPkg.Types { + if st, ok := t.Decl.Specs[0].(*ast.TypeSpec); ok { + if structType, ok := st.Type.(*ast.StructType); ok { + name := st.Name.Name + if visited[name] || skipTypes[name] { + continue + } + visited[name] = true + collectStruct(pkg, name, structType, t.Doc) + } + } + } + for _, file := range pkg.Syntax { + for _, decl := range file.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok || genDecl.Tok != token.CONST { + continue + } + + for _, spec := range genDecl.Specs { + vspec := spec.(*ast.ValueSpec) + for _, name := range vspec.Names { + obj := pkg.TypesInfo.Defs[name] + if obj == nil { + continue + } + typ := obj.Type().String() // e.g., "main.ErrCode" + parts := strings.Split(typ, ".") + typeName := parts[len(parts)-1] // just "ErrCode" + + if !includeConsts[typeName] { + continue + } + + if !rendered[typeName] { + constMap[typeName] = []constEntry{} + rendered[typeName] = true + } + + val := "" + if con, ok := obj.(*types.Const); ok { + val = con.Val().ExactString() + } + cdoc := strings.TrimSpace(vspec.Doc.Text()) + constMap[typeName] = append(constMap[typeName], constEntry{ + Name: name.Name, + Value: val, + Doc: cdoc, + }) + } + } + } + } + } + + writeOutput(output) +} + +func collectStruct(pkg *packages.Package, name string, structType *ast.StructType, docText string) { + info := StructInfo{ + Name: name, + Doc: strings.TrimSpace(docText), + } + + for _, field := range structType.Fields.List { + var fieldName string + if len(field.Names) > 0 { + fieldName = field.Names[0].Name + } else { + fieldName = fmt.Sprintf("%s", field.Type) + } + + var fieldType string + //if typ := pkg.TypesInfo.TypeOf(field.Type); typ != nil { + // fieldType = types.TypeString(typ, func(p *types.Package) string { + // return p.Name() + // }) + //} else { + // fieldType = fmt.Sprintf("%s", field.Type) + //} + var typ types.Type + if t := pkg.TypesInfo.TypeOf(field.Type); t != nil { + typ = t + fieldType = types.TypeString(t, func(p *types.Package) string { + return p.Name() + }) + } + + fieldTag := "" + if field.Tag != nil { + fieldTag = field.Tag.Value + } + + var fieldDoc string + if field.Doc != nil { + lines := strings.Split(field.Doc.Text(), "\n") + for i := range lines { + lines[i] = strings.TrimSpace(lines[i]) + } + fieldDoc = strings.Join(lines, " ") + } + + info.Fields = append(info.Fields, FieldInfo{ + Name: fieldName, + Type: fieldType, + Tag: fieldTag, + Doc: fieldDoc, + Typ: typ, + }) + + baseType := fieldType + + baseType = strings.TrimPrefix(baseType, "*") + + baseType = strings.TrimPrefix(baseType, "[]") + baseType = strings.Split(baseType, ".")[0] + if skipTypes[baseType] { + continue + } + if !visited[baseType] { + visited[baseType] = true + collectFromImports(baseType) + } + } + + structMap[name] = info +} + +func collectFromImports(typeName string) { + // future: support nested imports with doc.New(...) +} + +func writeOutput(path string) { + var buf bytes.Buffer + + buf.WriteString("# Storage Market Interface\n\n") + buf.WriteString("This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider.\n\n") + + buf.WriteString("## \U0001F4E1 MK20 HTTP API Overview\n\n") + buf.WriteString("The MK20 storage market module provides a set of HTTP endpoints under `/market/mk20` that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior.\n\n") + + buf.WriteString("### Base URL\n\n" + + "The base URL for all MK20 endpoints is: \n\n" + + "```\n\n/market/mk20\n\n```" + + "\n\n") + + buf.WriteString("### šŸ”„ POST /store\n\n") + buf.WriteString("Submit a new MK20 deal.\n\n") + buf.WriteString("- **Content-Type**: N/A\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**: N/A\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: Deal accepted\n") + buf.WriteString(" - Other [HTTP codes](#constants-for-errorcode) indicate validation failure, rejection, or system errors\n\n") + + buf.WriteString("### šŸ§¾ GET /status?id=\n\n") + buf.WriteString("Retrieve the current status of a deal.\n\n") + buf.WriteString("- **Content-Type**: `application/json`\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**:\n") + buf.WriteString(" - `id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: JSON-encoded [deal status](#dealstatusresponse) information\n") + buf.WriteString(" - `400 Bad Request`: Missing or invalid ID\n") + buf.WriteString(" - `500 Internal Server Error`: If backend fails to respond\n\n") + + buf.WriteString("### šŸ“œ GET /contracts\n\n") + buf.WriteString("- **Content-Type**: N/A\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**: N/A\n") + buf.WriteString("Return the list of contract addresses supported by the provider.\n\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: [JSON array of contract addresses](#supportedcontracts)\n") + buf.WriteString(" - `500 Internal Server Error`: Query or serialization failure\n\n") + + buf.WriteString("### šŸ—‚ PUT /data?id=\n\n") + buf.WriteString("Upload deal data after the deal has been accepted.\n\n") + buf.WriteString("- **Content-Type**: `application/octet-stream`\n") + buf.WriteString("- **Body**: Deal data bytes\n") + buf.WriteString("- **Query Parameter**:\n -`id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") + buf.WriteString("- **Headers**:\n") + buf.WriteString(" - `Content-Length`: must be deal's raw size\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: if data is successfully streamed\n") + buf.WriteString(" - `400`, `413`, or `415`: on validation failures\n\n") + + buf.WriteString("### šŸ§  GET /info\n\n") + buf.WriteString("- **Content-Type**: N/A\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**: N/A\n") + buf.WriteString("Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources.\n\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: with markdown content of the info file\n") + buf.WriteString(" - `500 Internal Server Error`: if file is not found or cannot be read\n\n") + + buf.WriteString("## Supported Deal Types\n\n") + buf.WriteString("This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.\n\n") + + ordered := []string{"Deal", "DataSource", "Products"} + var rest []string + for k := range structMap { + if k != "Deal" && k != "DataSource" && k != "Products" { + rest = append(rest, k) + } + } + sort.Strings(rest) + keys := append(ordered, rest...) + + for _, k := range keys { + s, ok := structMap[k] + if !ok { + continue + } + buf.WriteString(fmt.Sprintf("### %s\n\n", s.Name)) + if s.Doc != "" { + buf.WriteString(s.Doc + "\n\n") + } + buf.WriteString("| Field | Type | Tag | Description |\n") + buf.WriteString("|-------|------|-----|-------------|\n") + for _, f := range s.Fields { + typeName := f.Type + linkTarget := "" + + // Strip common wrappers like pointer/star and slice + trimmed := strings.TrimPrefix(typeName, "*") + trimmed = strings.TrimPrefix(trimmed, "[]") + parts := strings.Split(trimmed, ".") + baseType := parts[len(parts)-1] + + if _, ok := structMap[baseType]; ok { + linkTarget = fmt.Sprintf("[%s](#%s)", f.Type, strings.ToLower(baseType)) + } else if _, ok := constMap[baseType]; ok { + linkTarget = fmt.Sprintf("[%s](#constants-for-%s)", f.Type, strings.ToLower(baseType)) + } else { + typ := f.Typ + if ptr, ok := typ.(*types.Pointer); ok { + typ = ptr.Elem() + } + if named, ok := typ.(*types.Named); ok && named.Obj() != nil && named.Obj().Pkg() != nil { + + pkgPath := named.Obj().Pkg().Path() + objName := named.Obj().Name() + linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/%s#%s)", typeName, pkgPath, objName) + } else if typ != nil && typ.String() == baseType { + linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, baseType) + } else if slice, ok := typ.(*types.Slice); ok { + elem := slice.Elem() + if basic, ok := elem.(*types.Basic); ok { + linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, basic.Name()) + } else { + linkTarget = typeName + } + } else { + linkTarget = typeName + } + } + + buf.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", + f.Name, linkTarget, strings.Trim(f.Tag, "`"), f.Doc)) + } + buf.WriteString("\n") + } + + // Render constants with sort order + for k, v := range constMap { + if len(v) == 0 { + continue + } + buf.WriteString(fmt.Sprintf("### Constants for %s\n\n", k)) + buf.WriteString("| Constant | Code | Description |\n") + buf.WriteString("|----------|------|-------------|\n") + for _, c := range v { + buf.WriteString(fmt.Sprintf("| %s | %s | %s |\n", c.Name, c.Value, c.Doc)) + } + buf.WriteString("\n") + } + + err := os.WriteFile(path, buf.Bytes(), 0644) + if err != nil { + log.Fatalf("Failed to write output: %v", err) + } +} diff --git a/market/mk20/types.go b/market/mk20/types.go index 1a0e22166..fc8115778 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -24,6 +24,7 @@ type Deal struct { } type Products struct { + // DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) DDOV1 *DDOV1 `json:"ddov1"` } @@ -69,6 +70,7 @@ type PieceDataFormat struct { // FormatCar represents the CAR (Content Addressable aRchive) format with version metadata for piece data serialization. type FormatCar struct { + // Version specifies the version of the CAR format used for piece data serialization. Version uint64 `json:"version"` } @@ -88,6 +90,7 @@ type FormatBytes struct{} // DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. type DataSourceOffline struct { + // RawSize specifies the raw size of the data in bytes. RawSize uint64 `json:"rawsize"` } @@ -95,7 +98,7 @@ func (dso *DataSourceOffline) Name() DataSourceName { return DataSourceNameOffline } -func (dso *DataSourceOffline) IsEnabled(dbDataSources []dbDataSource) (int, error) { +func (dso *DataSourceOffline) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { name := string(dso.Name()) for _, p := range dbDataSources { if p.Name == name { @@ -116,7 +119,7 @@ func (dsa *DataSourceAggregate) Name() DataSourceName { return DataSourceNameAggregate } -func (dsa *DataSourceAggregate) IsEnabled(dbDataSources []dbDataSource) (int, error) { +func (dsa *DataSourceAggregate) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { name := string(dsa.Name()) for _, p := range dbDataSources { if p.Name == name { @@ -142,7 +145,7 @@ func (dsh *DataSourceHTTP) Name() DataSourceName { return DataSourceNameHTTP } -func (dsh *DataSourceHTTP) IsEnabled(dbDataSources []dbDataSource) (int, error) { +func (dsh *DataSourceHTTP) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { name := string(dsh.Name()) for _, p := range dbDataSources { if p.Name == name { @@ -170,7 +173,9 @@ type HttpUrl struct { Fallback bool `json:"fallback"` } +// DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted. type DataSourceHttpPut struct { + // RawSize specifies the raw size of the data in bytes. RawSize uint64 `json:"rawsize"` } @@ -178,7 +183,7 @@ func (dsh *DataSourceHttpPut) Name() DataSourceName { return DataSourceNamePut } -func (dsh *DataSourceHttpPut) IsEnabled(dbDataSources []dbDataSource) (int, error) { +func (dsh *DataSourceHttpPut) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { name := string(dsh.Name()) for _, p := range dbDataSources { if p.Name == name { @@ -194,30 +199,61 @@ func (dsh *DataSourceHttpPut) IsEnabled(dbDataSources []dbDataSource) (int, erro type AggregateType uint64 const ( + + // AggregateTypeNone represents the default aggregation type, indicating no specific aggregation is applied. AggregateTypeNone AggregateType = iota + + // AggregateTypeV1 represents the first version of the aggregate type in the system. AggregateTypeV1 ) -type ErrCode int +// ErrorCode represents an error code as an integer value +type ErrorCode int const ( - Ok = 200 - ErrBadProposal = 400 - ErrMalformedDataSource = 430 - ErrUnsupportedDataSource = 422 - ErrUnsupportedProduct = 423 - ErrProductNotEnabled = 424 - ErrProductValidationFailed = 425 - ErrDealRejectedByMarket = 426 - ErrServiceMaintenance = 503 - ErrServiceOverloaded = 429 - ErrMarketNotEnabled = 440 - ErrDurationTooShort = 441 + + // Ok represents a successful operation with an HTTP status code of 200. + Ok ErrorCode = 200 + + // ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic. + ErrBadProposal ErrorCode = 400 + + // ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data. + ErrMalformedDataSource ErrorCode = 430 + + // ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context. + ErrUnsupportedDataSource ErrorCode = 422 + + // ErrUnsupportedProduct indicates that the requested product is not supported by the provider. + ErrUnsupportedProduct ErrorCode = 423 + + // ErrProductNotEnabled indicates that the requested product is not enabled on the provider. + ErrProductNotEnabled ErrorCode = 424 + + // ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data. + ErrProductValidationFailed ErrorCode = 425 + + // ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules. + ErrDealRejectedByMarket ErrorCode = 426 + + // ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503. + ErrServiceMaintenance ErrorCode = 503 + + // ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment. + ErrServiceOverloaded ErrorCode = 429 + + // ErrMarketNotEnabled indicates that the market is not enabled for the requested operation. + ErrMarketNotEnabled ErrorCode = 440 + + // ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold. + ErrDurationTooShort ErrorCode = 441 ) +// ProductName represents a type for defining the product name identifier used in various operations and validations. type ProductName string const ( + // ProductNameDDOV1 represents the identifier for the "ddov1" product used in contract operations and validations. ProductNameDDOV1 ProductName = "ddov1" ) @@ -231,14 +267,3 @@ const ( DataSourceNamePDP DataSourceName = "pdp" DataSourceNamePut DataSourceName = "put" ) - -type dbDataSource struct { - Name string `db:"name"` - Enabled bool `db:"enabled"` -} - -// TODO: Client facing UI Page for SP -// TODO: Contract SP details pathway - sptool? -// TODO: SPID data source -// TODO: Test contract -// TODO: ACLv1? diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go index 3d434e110..adf2d4599 100644 --- a/market/mk20/types_test.go +++ b/market/mk20/types_test.go @@ -3,11 +3,12 @@ package mk20 import ( "testing" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/require" ) var ( @@ -81,7 +82,7 @@ func TestValidate_DDOV1(t *testing.T) { name string prod []dbProduct mutate func(*DDOV1) - wantCode int + wantCode ErrorCode }{ // enabled / disabled / unsupported {"no products on provider", @@ -156,7 +157,7 @@ func TestValidate_DataSource(t *testing.T) { name string mutateDS func(*DataSource) mutateDBSrc func([]dbDataSource) []dbDataSource - wantCode int + wantCode ErrorCode }{ // providerā€‘level enable / disable checks {"no data sources enabled", @@ -271,7 +272,7 @@ func TestValidate_Deal(t *testing.T) { tests := []struct { name string mutate func(*Deal, *productAndDataSource) - wantCode int + wantCode ErrorCode }{ {"happy path", func(d *Deal, _ *productAndDataSource) {}, diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 39750704c..729b02459 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -9,7 +9,6 @@ import ( "net/url" "time" - "github.com/filecoin-project/go-state-types/crypto" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "golang.org/x/xerrors" @@ -21,12 +20,17 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" ) +type dbDataSource struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` +} + type productAndDataSource struct { Products []dbProduct Data []dbDataSource } -func (d *Deal) Validate(pad *productAndDataSource) (int, error) { +func (d *Deal) Validate(pad *productAndDataSource) (ErrorCode, error) { code, err := d.Products.Validate(pad.Products) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) @@ -35,7 +39,7 @@ func (d *Deal) Validate(pad *productAndDataSource) (int, error) { return d.Data.Validate(pad.Data) } -func (d DataSource) Validate(dbDataSources []dbDataSource) (int, error) { +func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { if len(dbDataSources) == 0 { return ErrUnsupportedDataSource, xerrors.Errorf("no data sources enabled on the provider") } @@ -272,7 +276,7 @@ type dbProduct struct { Enabled bool `db:"enabled"` } -func (d Products) Validate(dbProducts []dbProduct) (int, error) { +func (d Products) Validate(dbProducts []dbProduct) (ErrorCode, error) { if len(dbProducts) == 0 { return ErrProductNotEnabled, xerrors.Errorf("no products enabled on the provider") } @@ -486,33 +490,53 @@ type ProviderDealRejectionInfo struct { Reason string } -type DealStatusRequest struct { - Identifier ulid.ULID `json:"identifier"` - Signature crypto.Signature `json:"signature"` -} - +// DealStatusResponse represents the response of a deal's status, including its current state and an optional error message. type DealStatusResponse struct { - State DealState `json:"status"` - ErrorMsg string `json:"errormsg"` + + // State indicates the current processing state of the deal as a DealState value. + State DealState `json:"status"` + + // ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. + ErrorMsg string `json:"errormsg"` } +// DealStatus represents the status of a deal, including the HTTP code and an optional response detailing the deal's state and error message. type DealStatus struct { + + // Response provides details about the deal's status, such as its current state and any associated error messages, if available. Response *DealStatusResponse + + // HTTPCode represents the HTTP status code providing additional context about the deal status or possible errors. HTTPCode int } +// DealState represents the current status of a deal in the system as a string value. type DealState string const ( - DealStateAccepted DealState = "accepted" + + // DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system. + DealStateAccepted DealState = "accepted" + + // DealStateProcessing represents the state of a deal currently being processed in the pipeline. DealStateProcessing DealState = "processing" - DealStateSealing DealState = "sealing" - DealStateIndexing DealState = "indexing" - DealStateFailed DealState = "failed" - DealStateComplete DealState = "complete" + + // DealStateSealing indicates that the deal is currently being sealed in the system. + DealStateSealing DealState = "sealing" + + // DealStateIndexing represents the state where a deal is undergoing indexing in the system. + DealStateIndexing DealState = "indexing" + + // DealStateFailed indicates that the deal has failed due to an error during processing, sealing, or indexing. + DealStateFailed DealState = "failed" + + // DealStateComplete indicates that the deal has successfully completed all processing and is finalized in the system. + DealStateComplete DealState = "complete" ) +// SupportedContracts represents a collection of contract addresses supported by a system or application. type SupportedContracts struct { + // Contracts represents a list of supported contract addresses in string format. Contracts []string `json:"contracts"` } diff --git a/pdp/handlers.go b/pdp/handlers.go index 1ea516f54..a64b47a4d 100644 --- a/pdp/handlers.go +++ b/pdp/handlers.go @@ -33,9 +33,6 @@ import ( types2 "github.com/filecoin-project/lotus/chain/types" ) -// PDPRoutePath is the base path for PDP routes -const PDPRoutePath = "/pdp" - // PDPService represents the service for managing proof sets and pieces type PDPService struct { db *harmonydb.DB @@ -63,9 +60,12 @@ func NewPDPService(db *harmonydb.DB, stor paths.StashStore, ec *ethclient.Client } // Routes registers the HTTP routes with the provided router -func Routes(r *chi.Mux, p *PDPService) { +func Routes(p *PDPService) http.Handler { + + r := chi.NewRouter() + // Routes for proof sets - r.Route(path.Join(PDPRoutePath, "/proof-sets"), func(r chi.Router) { + r.Route("/proof-sets", func(r chi.Router) { // POST /pdp/proof-sets - Create a new proof set r.Post("/", p.handleCreateProofSet) @@ -100,17 +100,18 @@ func Routes(r *chi.Mux, p *PDPService) { }) }) - r.Get(path.Join(PDPRoutePath, "/ping"), p.handlePing) + r.Get("/ping", p.handlePing) // Routes for piece storage and retrieval // POST /pdp/piece - r.Post(path.Join(PDPRoutePath, "/piece"), p.handlePiecePost) + r.Post("/piece", p.handlePiecePost) // GET /pdp/piece/ - r.Get(path.Join(PDPRoutePath, "/piece/"), p.handleFindPiece) + r.Get("/piece/", p.handleFindPiece) // PUT /pdp/piece/upload/{uploadUUID} - r.Put(path.Join(PDPRoutePath, "/piece/upload/{uploadUUID}"), p.handlePieceUpload) + r.Put("/piece/upload/{uploadUUID}", p.handlePieceUpload) + return r } // Handler functions diff --git a/pdp/handlers_upload.go b/pdp/handlers_upload.go index ebef674ba..c53c5cbe7 100644 --- a/pdp/handlers_upload.go +++ b/pdp/handlers_upload.go @@ -205,7 +205,7 @@ func (p *PDPService) handlePiecePost(w http.ResponseWriter, r *http.Request) { } // Create a location URL where the piece data can be uploaded via PUT - uploadURL = path.Join(PDPRoutePath, "/piece/upload", uploadUUID.String()) + uploadURL = path.Join(r.URL.Path, "upload", uploadUUID.String()) responseStatus = http.StatusCreated return true, nil // Commit the transaction From f442ac78539437899d9e133ab8c1d9e7b6586358 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 20 May 2025 17:34:25 +0400 Subject: [PATCH 05/55] client, commv2, ui --- cmd/curio/tasks/tasks.go | 5 +- cmd/sptool/main.go | 1 + cmd/sptool/toolbox_deal_client.go | 253 ++++++++++++- go.mod | 65 ++-- go.sum | 140 ++++---- .../harmonydb/sql/20250505-market_mk20.sql | 75 +++- lib/cachedreader/cachedreader.go | 20 +- lib/commcidv2/commcidv2.go | 175 +++++++++ market/indexstore/indexstore.go | 64 ++-- market/indexstore/indexstore_test.go | 4 +- market/ipni/chunker/serve-chunker.go | 47 ++- market/mk20/ddo_v1.go | 57 ++- market/mk20/mk20.go | 34 +- market/mk20/types.go | 99 +----- market/mk20/types_test.go | 331 ----------------- market/mk20/utils.go | 165 ++++++--- market/retrieval/piecehandler.go | 13 +- .../remoteblockstore/remoteblockstore.go | 18 +- tasks/indexing/task_indexing.go | 333 +++++++++++++++--- tasks/indexing/task_ipni.go | 137 +++++-- tasks/pdp/task_prove.go | 2 +- tasks/storage-market/mk20.go | 4 +- web/api/webrpc/deals.go | 13 + web/api/webrpc/ipni.go | 10 +- web/api/webrpc/market.go | 265 ++++++++++++-- web/api/webrpc/market_20.go | 89 +++++ web/api/webrpc/sector.go | 20 ++ web/static/pages/ipni/ipni_search.mjs | 2 +- web/static/pages/market/index.html | 16 - web/static/pages/market/pending-deals.mjs | 2 +- web/static/pages/mk12-deal/deal.mjs | 2 +- web/static/pages/mk12-deal/index.html | 2 +- .../{market => mk12-deals}/deal-pipelines.mjs | 6 +- web/static/pages/mk12-deals/index.html | 16 + .../{market => mk12-deals}/market-asks.mjs | 100 +++--- web/static/pages/mk12-deals/mk12-deals.mjs | 2 +- web/static/pages/mk12-deals/mk12ddo-list.mjs | 3 +- web/static/pages/mk20-deal/deal.mjs | 256 ++++++++++++++ web/static/pages/mk20-deal/index.html | 36 ++ web/static/pages/mk20/ddo.mjs | 177 ++++++++++ web/static/pages/mk20/index.html | 25 ++ web/static/pages/sector/sector-info.mjs | 2 +- web/static/ux/curio-ux.mjs | 10 +- 43 files changed, 2200 insertions(+), 896 deletions(-) create mode 100644 lib/commcidv2/commcidv2.go delete mode 100644 market/mk20/types_test.go create mode 100644 web/api/webrpc/market_20.go rename web/static/pages/{market => mk12-deals}/deal-pipelines.mjs (98%) rename web/static/pages/{market => mk12-deals}/market-asks.mjs (83%) create mode 100644 web/static/pages/mk20-deal/deal.mjs create mode 100644 web/static/pages/mk20-deal/index.html create mode 100644 web/static/pages/mk20/ddo.mjs create mode 100644 web/static/pages/mk20/index.html diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index f1ed36a2c..b1a47579f 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -99,7 +99,6 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan machine := dependencies.ListenAddr prover := dependencies.Prover iStore := dependencies.IndexStore - pp := dependencies.SectorReader var activeTasks []harmonytask.TaskInterface @@ -295,8 +294,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan idxMax := taskhelp.Max(cfg.Subsystems.IndexingMaxTasks) - indexingTask := indexing.NewIndexingTask(db, sc, iStore, pp, cfg, idxMax) - ipniTask := indexing.NewIPNITask(db, sc, iStore, pp, cfg, idxMax) + indexingTask := indexing.NewIndexingTask(db, sc, iStore, dependencies.CachedPieceReader, cfg, idxMax) + ipniTask := indexing.NewIPNITask(db, sc, iStore, dependencies.CachedPieceReader, cfg, idxMax) activeTasks = append(activeTasks, ipniTask, indexingTask) if cfg.HTTP.Enable { diff --git a/cmd/sptool/main.go b/cmd/sptool/main.go index dd3d00e9e..6fccb4c09 100644 --- a/cmd/sptool/main.go +++ b/cmd/sptool/main.go @@ -101,5 +101,6 @@ var toolboxCmd = &cli.Command{ Subcommands: []*cli.Command{ sparkCmd, mk12Clientcmd, + mk20Clientcmd, }, } diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 05e53bb21..bdd8993cd 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -19,6 +19,7 @@ import ( "time" "github.com/dustin/go-humanize" + "github.com/filecoin-project/curio/market/mk20" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/maurl" @@ -42,7 +43,6 @@ import ( "github.com/filecoin-project/curio/lib/keystore" mk12_libp2p "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/mk12" - "github.com/filecoin-project/lotus/api" chain_types "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" @@ -1561,3 +1561,254 @@ var dealStatusCmd = &cli.Command{ return nil }, } + +var mk20Clientcmd = &cli.Command{ + Name: "mk20-client", + Usage: "mk20 client for Curio", + Flags: []cli.Flag{ + mk12_client_repo, + }, + Subcommands: []*cli.Command{ + initCmd, + mk20DealCmd, + }, +} + +var mk20DealCmd = &cli.Command{ + Name: "deal", + Usage: "Make a mk20 deal with Curio", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "http-url", + Usage: "http url to CAR file", + Required: true, + }, + &cli.StringSliceFlag{ + Name: "http-headers", + Usage: "http headers to be passed with the request (e.g key=value)", + }, + &cli.Uint64Flag{ + Name: "car-size", + Usage: "size of the CAR file: required for online deals", + Required: true, + }, + &cli.StringFlag{ + Name: "provider", + Usage: "storage provider on-chain address", + Required: true, + }, + &cli.StringFlag{ + Name: "commp", + Usage: "commp of the CAR file", + Required: true, + }, + &cli.Uint64Flag{ + Name: "piece-size", + Usage: "size of the CAR file as a padded piece", + Required: true, + }, + &cli.IntFlag{ + Name: "duration", + Usage: "duration of the deal in epochs", + Value: 518400, // default is 2880 * 180 == 180 days + }, + &cli.BoolFlag{ + Name: "verified", + Usage: "whether the deal funds should come from verified client data-cap", + Value: false, + }, + &cli.BoolFlag{ + Name: "indexing", + Usage: "indicates that an deal should be indexed", + Value: true, + }, + &cli.StringFlag{ + Name: "wallet", + Usage: "wallet address to be used to initiate the deal", + }, + &cli.BoolFlag{ + Name: "announce", + Usage: "indicates that deal should be announced to the IPNI(Network Indexer)", + Value: true, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + n, err := Setup(cctx.String(mk12_client_repo.Name)) + if err != nil { + return err + } + + api, closer, err := lcli.GetGatewayAPIV1(cctx) + if err != nil { + return fmt.Errorf("cant setup gateway connection: %w", err) + } + defer closer() + if err != nil { + return xerrors.Errorf("cant setup gateway connection: %w", err) + } + defer closer() + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return err + } + + log.Debugw("selected wallet", "wallet", walletAddr) + + maddr, err := address.NewFromString(cctx.String("provider")) + if err != nil { + return err + } + + minfo, err := api.StateMinerInfo(ctx, maddr, chain_types.EmptyTSK) + if err != nil { + return err + } + if minfo.PeerId == nil { + return xerrors.Errorf("storage provider %s has no peer ID set on-chain", maddr) + } + + var maddrs []multiaddr.Multiaddr + for _, mma := range minfo.Multiaddrs { + ma, err := multiaddr.NewMultiaddrBytes(mma) + if err != nil { + return xerrors.Errorf("storage provider %s had invalid multiaddrs in their info: %w", maddr, err) + } + maddrs = append(maddrs, ma) + } + if len(maddrs) == 0 { + return xerrors.Errorf("storage provider %s has no multiaddrs set on-chain", maddr) + } + + addrInfo := &peer.AddrInfo{ + ID: *minfo.PeerId, + Addrs: maddrs, + } + + log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr) + + var hurls []*url.URL + + for _, ma := range addrInfo.Addrs { + hurl, err := maurl.ToURL(ma) + if err != nil { + return xerrors.Errorf("failed to convert multiaddr %s to URL: %w", ma, err) + } + if hurl.Scheme == "ws" { + hurl.Scheme = "http" + } + if hurl.Scheme == "wss" { + hurl.Scheme = "https" + } + log.Debugw("converted multiaddr to URL", "url", hurl, "multiaddr", ma.String()) + hurls = append(hurls, hurl) + } + + commp := cctx.String("commp") + pieceCid, err := cid.Parse(commp) + if err != nil { + return xerrors.Errorf("parsing commp '%s': %w", commp, err) + } + + pieceSize := cctx.Uint64("piece-size") + if pieceSize == 0 { + return xerrors.Errorf("must provide piece-size parameter for CAR url") + } + + carFileSize := cctx.Uint64("car-size") + if carFileSize == 0 { + return xerrors.Errorf("size of car file cannot be 0") + } + + url, err := url.Parse(cctx.String("http-url")) + if err != nil { + return xerrors.Errorf("parsing http url: %w", err) + } + + var headers http.Header + + for _, header := range cctx.StringSlice("http-headers") { + sp := strings.Split(header, "=") + if len(sp) != 2 { + return xerrors.Errorf("malformed http header: %s", header) + } + headers.Add(sp[0], sp[1]) + } + + d := mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + RawSize: carFileSize, + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Headers: headers, + Priority: 0, + Fallback: true, + }, + }, + }, + } + + p := mk20.Products{ + DDOV1: &mk20.DDOV1{ + Provider: maddr, + Client: walletAddr, + PieceManager: walletAddr, + Duration: abi.ChainEpoch(cctx.Int64("duration")), + ContractAddress: cctx.String("contract-address"), + ContractVerifyMethod: cctx.String("contract-verify-method"), + ContractVerifyMethodParams: []byte("test bytes"), + Indexing: cctx.Bool("indexing"), + AnnounceToIPNI: cctx.Bool("announce"), + }, + } + + id, err := mk20.NewULID() + if err != nil { + return err + } + log.Debugw("generated deal id", "id", id) + + deal := mk20.Deal{ + Identifier: id, + Data: d, + Products: p, + } + + log.Debugw("deal", "deal", deal) + + body, err := json.Marshal(deal) + if err != nil { + return err + } + + // Try to request all URLs one by one and exit after first success + for _, u := range hurls { + s := u.String() + "/market/mk20/store" + log.Debugw("trying to send request to", "url", u.String()) + req, err := http.NewRequest("POST", s, bytes.NewReader(body)) + if err != nil { + return xerrors.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/cbor") + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Warnw("failed to send request", "url", s, "error", err) + continue + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + log.Warnw("failed to send request", "url", s, "status", resp.StatusCode, "body", resp.Body) + continue + } + return nil + } + return xerrors.Errorf("failed to send request to any of the URLs") + }, +} diff --git a/go.mod b/go.mod index 5cee3cb6d..9e8e9533f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/filecoin-project/filecoin-ffi v1.32.0 github.com/filecoin-project/go-address v1.2.0 github.com/filecoin-project/go-bitfield v0.2.4 - github.com/filecoin-project/go-cbor-util v0.0.1 + github.com/filecoin-project/go-cbor-util v0.0.2 github.com/filecoin-project/go-commp-utils v0.1.4 github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 @@ -29,7 +29,7 @@ require ( github.com/filecoin-project/go-f3 v0.8.4 github.com/filecoin-project/go-fil-commcid v0.2.0 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 - github.com/filecoin-project/go-jsonrpc v0.7.0 + github.com/filecoin-project/go-jsonrpc v0.7.1 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-state-types v0.16.0 github.com/filecoin-project/go-statestore v0.2.0 @@ -53,9 +53,9 @@ require ( github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 + github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5 github.com/invopop/jsonschema v0.12.0 - github.com/ipfs/boxo v0.21.0 + github.com/ipfs/boxo v0.24.3 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 @@ -67,7 +67,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/frisbii v0.6.1 github.com/ipld/go-car v0.6.2 - github.com/ipld/go-car/v2 v2.13.1 + github.com/ipld/go-car/v2 v2.14.2 github.com/ipld/go-ipld-prime v0.21.0 github.com/ipni/go-libipni v0.6.13 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 @@ -88,10 +88,10 @@ require ( github.com/oklog/ulid v1.3.1 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_golang v1.21.1 github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 - github.com/samber/lo v1.39.0 + github.com/samber/lo v1.47.0 github.com/schollz/progressbar/v3 v3.18.0 github.com/sirupsen/logrus v1.9.3 github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb @@ -105,7 +105,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.36.0 - golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac + golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa golang.org/x/net v0.38.0 golang.org/x/sync v0.12.0 golang.org/x/sys v0.31.0 @@ -125,9 +125,8 @@ require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/StackExchange/wmi v1.2.1 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/akavel/rsrc v0.8.0 // indirect + github.com/akavel/rsrc v0.10.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect @@ -144,7 +143,7 @@ require ( github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect - github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -152,7 +151,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect @@ -168,7 +167,7 @@ require ( github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect - github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect + github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 // indirect github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/filecoin-project/go-crypto v0.1.0 // indirect @@ -186,6 +185,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/gdamore/tcell/v2 v2.2.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -201,7 +201,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.4 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/gopacket v1.1.19 // indirect @@ -231,8 +231,8 @@ require ( github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-peertaskqueue v0.8.1 // indirect - github.com/ipfs/go-unixfsnode v1.9.0 // indirect + github.com/ipfs/go-peertaskqueue v0.8.2 // indirect + github.com/ipfs/go-unixfsnode v1.9.2 // indirect github.com/ipfs/go-verifcid v0.0.3 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20240322071803-376decb85801 // indirect @@ -244,22 +244,22 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/jessevdk/go-flags v1.4.0 // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/koron/go-ssdp v0.0.5 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.25.2 // indirect - github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect github.com/libp2p/go-libp2p-pubsub v0.13.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect github.com/libp2p/go-maddr-filter v0.1.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect @@ -267,7 +267,7 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.2 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/magefile/mage v1.9.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.14 // indirect @@ -288,7 +288,7 @@ require ( github.com/multiformats/go-multistream v0.6.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nikkolasg/hexjson v0.1.0 // indirect - github.com/nkovacs/streamquote v1.0.0 // indirect + github.com/nkovacs/streamquote v1.1.0 // indirect github.com/onsi/ginkgo/v2 v2.22.2 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -298,7 +298,7 @@ require ( github.com/pion/dtls/v2 v2.2.12 // indirect github.com/pion/dtls/v3 v3.0.4 // indirect github.com/pion/ice/v2 v2.3.37 // indirect - github.com/pion/ice/v4 v4.0.6 // indirect + github.com/pion/ice/v4 v4.0.8 // indirect github.com/pion/interceptor v0.1.39 // indirect github.com/pion/logging v0.2.3 // indirect github.com/pion/mdns v0.0.12 // indirect @@ -306,7 +306,7 @@ require ( github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.15 // indirect github.com/pion/rtp v1.8.18 // indirect - github.com/pion/sctp v1.8.35 // indirect + github.com/pion/sctp v1.8.37 // indirect github.com/pion/sdp/v3 v3.0.10 // indirect github.com/pion/srtp/v3 v3.0.4 // indirect github.com/pion/stun v0.6.1 // indirect @@ -315,7 +315,7 @@ require ( github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v2 v2.1.6 // indirect github.com/pion/turn/v4 v4.0.0 // indirect - github.com/pion/webrtc/v4 v4.0.9 // indirect + github.com/pion/webrtc/v4 v4.0.10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -323,20 +323,20 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.49.0 // indirect + github.com/quic-go/quic-go v0.50.1 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect github.com/twmb/murmur3 v1.1.6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasttemplate v1.0.1 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect @@ -345,6 +345,7 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-filecoin-go v1.0.1 // indirect @@ -380,7 +381,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect - lukechampine.com/blake3 v1.3.0 // indirect + lukechampine.com/blake3 v1.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index beff2c37e..7c6765737 100644 --- a/go.sum +++ b/go.sum @@ -73,16 +73,15 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= -github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/akavel/rsrc v0.10.2 h1:Zxm8V5eI1hW4gGaYsJQUhxpjkENuG91ki8B4zCrvEsw= +github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o= github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -91,8 +90,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= -github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -209,8 +208,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA= -github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn84wUS1pVikGiruW+x5XM4amhjaZO20vCjay4= +github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= @@ -228,10 +227,10 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -294,8 +293,9 @@ github.com/filecoin-project/go-address v1.2.0/go.mod h1:kQEQ4qZ99a51X7DjT9HiMT4y github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UNxPeOYaXhBWSNsVaM= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o= -github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= +github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 h1:n+nczYe6VedXmdtAXygRuey246YnYyuY1NPrmy2iK6s= +github.com/filecoin-project/go-amt-ipld/v3 v3.1.1/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= github.com/filecoin-project/go-amt-ipld/v4 v4.4.0 h1:6kvvMeSpIy4GTU5t3vPHZgWYIMRzGRKLJ73s/cltsoc= github.com/filecoin-project/go-amt-ipld/v4 v4.4.0/go.mod h1:msgmUxTyRBZ6iXt+5dnUDnIb7SEFqdPsbB1wyo/G3ts= @@ -304,8 +304,8 @@ github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQj github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk= github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= -github.com/filecoin-project/go-cbor-util v0.0.1 h1:E1LYZYTtjfAQwCReho0VXvbu8t3CYAVPiMx8EiV/VAs= -github.com/filecoin-project/go-cbor-util v0.0.1/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-cbor-util v0.0.2 h1:vljF+a+NBwv89VfPvy5lJEtrZWe8k4nizgaqWhf6Ro8= +github.com/filecoin-project/go-cbor-util v0.0.2/go.mod h1:96OIHk38Y1IV+KCXkGjE2WjjIxfpIanz2rWIIy5kKkQ= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk= @@ -337,8 +337,8 @@ github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGy github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 h1:nYs6OPUF8KbZ3E8o9p9HJnQaE8iugjHR5WYVMcicDJc= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0/go.mod h1:s0qiHRhFyrgW0SvdQMSJFQxNa4xEIG5XvqCBZUEgcbc= -github.com/filecoin-project/go-jsonrpc v0.7.0 h1:mqA5pIOlBODx7ascY9cJdBAYonhgbdUOIn2dyYI1YBg= -github.com/filecoin-project/go-jsonrpc v0.7.0/go.mod h1:lAUpS8BSVtKaA8+/CFUMA5dokMiSM7n0ehf8bHOFdpE= +github.com/filecoin-project/go-jsonrpc v0.7.1 h1:++oUd7R3aYibLKXS/DsO348Lco+1cJbfCwRiv8awHFQ= +github.com/filecoin-project/go-jsonrpc v0.7.1/go.mod h1:lAUpS8BSVtKaA8+/CFUMA5dokMiSM7n0ehf8bHOFdpE= github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs= github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8= @@ -394,8 +394,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gammazero/channelqueue v0.2.2 h1:ufNzIbeDBxNfHj0m5uwUfOwvTmHF/O40hu2ZNnvF+/8= github.com/gammazero/channelqueue v0.2.2/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= -github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= -github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4= github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= @@ -434,7 +434,7 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -475,8 +475,9 @@ github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -629,8 +630,8 @@ github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3 github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo= -github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 h1:9tcYMdi+7Rb1y0E9Del1DRHui7Ne3za5lLw6CjMJv/M= -github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ= +github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5 h1:FcxwOojw6pUiPpsf7Q6Fw/pI+7cR6FlapLBEGV/902A= +github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -639,8 +640,8 @@ github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uO github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.21.0 h1:XpGXb+TQQ0IUdYaeAxGzWjSs6ow/Lce148A/2IbRDVE= -github.com/ipfs/boxo v0.21.0/go.mod h1:NmweAYeY1USOaJJxouy7DLr/Y5M8UBSsCI2KRivO+TY= +github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c= +github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= @@ -693,8 +694,6 @@ github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHv github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= -github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= -github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -752,15 +751,13 @@ github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= -github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= -github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= +github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= github.com/ipfs/go-test v0.0.4 h1:DKT66T6GBB6PsDFLoO56QZPrOmzJkqU1FZH5C9ySkew= github.com/ipfs/go-test v0.0.4/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= -github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= -github.com/ipfs/go-unixfsnode v1.9.0 h1:ubEhQhr22sPAKO2DNsyVBW7YB/zA8Zkif25aBvz8rc8= -github.com/ipfs/go-unixfsnode v1.9.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= +github.com/ipfs/go-unixfsnode v1.9.2 h1:0A12BYs4XOtDPJTMlwmNPlllDfqcc4yie4e919hcUXk= +github.com/ipfs/go-unixfsnode v1.9.2/go.mod h1:v1nuMFHf4QTIhFUdPMvg1nQu7AqDLvIdwyvJ531Ot1U= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= @@ -769,8 +766,8 @@ github.com/ipld/frisbii v0.6.1/go.mod h1:5alsRVbOyUbZ2In70AdJ4VOLh13LkmAMUomotJa github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= -github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4= -github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= +github.com/ipld/go-car/v2 v2.14.2 h1:9ERr7KXpCC7If0rChZLhYDlyr6Bes6yRKPJnCO3hdHY= +github.com/ipld/go-car/v2 v2.14.2/go.mod h1:0iPB/825lTZLU2zPK5bVTk/R3V2612E1VI279OGSXWA= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-fixtureplate v0.0.3 h1:Qb/rBBnYP8IiK+VLq89y2NPZ3iQeQpAi9YK3oSleVGs= @@ -822,8 +819,9 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jellydator/ttlcache/v2 v2.11.1 h1:AZGME43Eh2Vv3giG6GeqeLeFXxwxn1/qHItqWZl6U64= github.com/jellydator/ttlcache/v2 v2.11.1/go.mod h1:RtE5Snf0/57e+2cLWFYWCCsLas2Hy3c5Z4n14XmSvTI= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -857,8 +855,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -909,10 +907,10 @@ github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= -github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= -github.com/libp2p/go-libp2p-kad-dht v0.25.2/go.mod h1:6za56ncRHYXX4Nc2vn8z7CZK0P4QiMcrn77acKLM2Oo= -github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= -github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= +github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= +github.com/libp2p/go-libp2p-kad-dht v0.28.1/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= +github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= +github.com/libp2p/go-libp2p-kbucket v0.6.4/go.mod h1:jp6w82sczYaBsAypt5ayACcRJi0lgsba7o4TzJKEfWA= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -925,8 +923,8 @@ github.com/libp2p/go-libp2p-pubsub v0.13.0/go.mod h1:m0gpUOyrXKXdE7c8FNQ9/HLfWbx github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.7.3 h1:u1LGzAMVRK9Nqq5aYDVOiq/HaB93U9WWczBzGyAC5ZY= -github.com/libp2p/go-libp2p-routing-helpers v0.7.3/go.mod h1:cN4mJAD/7zfPKXBcs9ze31JGYAZgzdABEm+q/hkswb8= +github.com/libp2p/go-libp2p-routing-helpers v0.7.4 h1:6LqS1Bzn5CfDJ4tzvP9uwh42IB7TJLNFJA6dEeGBv84= +github.com/libp2p/go-libp2p-routing-helpers v0.7.4/go.mod h1:we5WDj9tbolBXOuF1hGOkR+r7Uh1408tQbAKaT5n1LE= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -968,8 +966,9 @@ github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magik6k/reflink v1.0.2-patch1 h1:NXSgQugcESI8Z/jBtuAI83YsZuRauY9i9WOyOnJ7Vns= github.com/magik6k/reflink v1.0.2-patch1/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= @@ -1102,8 +1101,9 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nikkolasg/hexjson v0.1.0 h1:Cgi1MSZVQFoJKYeRpBNEcdF3LB+Zo4fYKsDz7h8uJYQ= github.com/nikkolasg/hexjson v0.1.0/go.mod h1:fbGbWFZ0FmJMFbpCMtJpwb0tudVxSSZ+Es2TsCg57cA= -github.com/nkovacs/streamquote v1.0.0 h1:PmVIV08Zlx2lZK5fFZlMZ04eHcDTIFJCv/5/0twVUow= github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= +github.com/nkovacs/streamquote v1.1.0 h1:wDY1+Hikdx4iOmZZBFLXvwLr7zj9uPIoXfijz+6ad2g= +github.com/nkovacs/streamquote v1.1.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= @@ -1153,8 +1153,8 @@ github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U= github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg= github.com/pion/ice/v2 v2.3.37 h1:ObIdaNDu1rCo7hObhs34YSBcO7fjslJMZV0ux+uZWh0= github.com/pion/ice/v2 v2.3.37/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= -github.com/pion/ice/v4 v4.0.6 h1:jmM9HwI9lfetQV/39uD0nY4y++XZNPhvzIPCb8EwxUM= -github.com/pion/ice/v4 v4.0.6/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= +github.com/pion/ice/v4 v4.0.8 h1:ajNx0idNG+S+v9Phu4LSn2cs8JEfTsA1/tEjkkAVpFY= +github.com/pion/ice/v4 v4.0.8/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= github.com/pion/interceptor v0.1.39 h1:Y6k0bN9Y3Lg/Wb21JBWp480tohtns8ybJ037AGr9UuA= github.com/pion/interceptor v0.1.39/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= @@ -1170,8 +1170,8 @@ github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= github.com/pion/rtp v1.8.18 h1:yEAb4+4a8nkPCecWzQB6V/uEU18X1lQCGAQCjP+pyvU= github.com/pion/rtp v1.8.18/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= -github.com/pion/sctp v1.8.35 h1:qwtKvNK1Wc5tHMIYgTDJhfZk7vATGVHhXbUDfHbYwzA= -github.com/pion/sctp v1.8.35/go.mod h1:EcXP8zCYVTRy3W9xtOF7wJm1L1aXfKRQzaM33SjQlzg= +github.com/pion/sctp v1.8.37 h1:ZDmGPtRPX9mKCiVXtMbTWybFw3z/hVKAZgU81wcOrqs= +github.com/pion/sctp v1.8.37/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA= github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= @@ -1192,8 +1192,8 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= -github.com/pion/webrtc/v4 v4.0.9 h1:PyOYMRKJgfy0dzPcYtFD/4oW9zaw3Ze3oZzzbj2LV9E= -github.com/pion/webrtc/v4 v4.0.9/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck= +github.com/pion/webrtc/v4 v4.0.10 h1:Hq/JLjhqLxi+NmCtE8lnRPDr8H4LcNvwg8OxVcdv56Q= +github.com/pion/webrtc/v4 v4.0.10/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1215,8 +1215,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1248,8 +1248,8 @@ github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Ez github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94= -github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= +github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= +github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= @@ -1270,14 +1270,14 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= -github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1367,10 +1367,10 @@ github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/triplewz/poseidon v0.0.2 h1:s5QMVYnUfqvgM1eIqp7O9hHjZLVrKnkhx0E7EQTf9Nk= github.com/triplewz/poseidon v0.0.2/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= @@ -1384,8 +1384,9 @@ github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g= github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -1444,6 +1445,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -1575,8 +1578,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= +golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1747,6 +1750,7 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2009,8 +2013,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= -lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/blake3 v1.4.0 h1:xDbKOZCVbnZsfzM6mHSYcGRHZ3YrLDzqz8XnV4uaD5w= +lukechampine.com/blake3 v1.4.0/go.mod h1:MQJNQCTnR+kwOP/JEZSxj3MaQjp80FOFSNMMHXcSeX0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 90d07f2cc..b9590fe72 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -1,3 +1,67 @@ +-- Drop the existing primary key constraint for market_piece_metadata +ALTER TABLE market_piece_metadata +DROP CONSTRAINT market_piece_metadata_pkey; + +-- Drop the redundant UNIQUE constraint if it exists for market_piece_metadata +ALTER TABLE market_piece_metadata +DROP CONSTRAINT IF EXISTS market_piece_meta_identity_key; + +-- Add the new composite primary key for market_piece_metadata +ALTER TABLE market_piece_metadata + ADD PRIMARY KEY (piece_cid, piece_size); + +-- Add ID column to ipni_task table +ALTER TABLE ipni_task + ADD COLUMN id TEXT; + +-- Function to create ipni tasks +CREATE OR REPLACE FUNCTION insert_ipni_task( + _id TEXT, + _sp_id BIGINT, + _sector BIGINT, + _reg_seal_proof INT, + _sector_offset BIGINT, + _context_id BYTEA, + _is_rm BOOLEAN, + _provider TEXT, + _task_id BIGINT DEFAULT NULL +) RETURNS VOID AS $$ +DECLARE +_existing_is_rm BOOLEAN; + _latest_is_rm BOOLEAN; +BEGIN + -- Check if ipni_task has the same context_id and provider with a different is_rm value + SELECT is_rm INTO _existing_is_rm + FROM ipni_task + WHERE provider = _provider AND context_id = _context_id AND is_rm != _is_rm + LIMIT 1; + + -- If a different is_rm exists for the same context_id and provider, insert the new task + IF FOUND THEN + INSERT INTO ipni_task (sp_id, id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); + RETURN; + END IF; + + -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table + SELECT is_rm INTO _latest_is_rm + FROM ipni + WHERE provider = _provider AND context_id = _context_id + ORDER BY order_number DESC + LIMIT 1; + + -- If the latest ad has the same is_rm value, raise an exception + IF FOUND AND _latest_is_rm = _is_rm THEN + RAISE EXCEPTION 'already published'; + END IF; + + -- If all conditions are met, insert the new task into ipni_task + INSERT INTO ipni_task (sp_id, id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); + END; + $$ LANGUAGE plpgsql; + + CREATE TABLE ddo_contracts ( address TEXT NOT NULL PRIMARY KEY, abi TEXT NOT NULL @@ -5,6 +69,9 @@ CREATE TABLE ddo_contracts ( CREATE TABLE market_mk20_deal ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + + sp_id BIGINT NOT NULL, + id TEXT PRIMARY KEY, piece_cid TEXT NOT NULL, size BIGINT NOT NULL, @@ -13,8 +80,9 @@ CREATE TABLE market_mk20_deal ( source_http JSONB NOT NULL DEFAULT 'null', source_aggregate JSONB NOT NULL DEFAULT 'null', source_offline JSONB NOT NULL DEFAULT 'null', + source_http_put JSONB NOT NULL DEFAULT 'null', - ddov1 JSONB NOT NULL DEFAULT 'null', + ddo_v1 JSONB NOT NULL DEFAULT 'null', market_deal_id TEXT DEFAULT NULL, error TEXT DEFAULT NULL @@ -89,8 +157,7 @@ CREATE TABLE market_mk20_offline_urls ( PRIMARY KEY (id, piece_cid, piece_size), CONSTRAINT market_mk20_offline_urls_id_fk FOREIGN KEY (id) REFERENCES market_mk20_pipeline (id) - ON DELETE CASCADE, - CONSTRAINT market_mk20_offline_urls_id_unique UNIQUE (id) + ON DELETE CASCADE ); CREATE TABLE market_mk20_products ( @@ -103,7 +170,7 @@ CREATE TABLE market_mk20_data_source ( enabled BOOLEAN DEFAULT TRUE ); -INSERT INTO market_mk20_products (name, enabled) VALUES ('ddov1', TRUE); +INSERT INTO market_mk20_products (name, enabled) VALUES ('ddo_v1', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('http', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index caf78c050..16edd1018 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -125,7 +125,7 @@ func (r *cachedSectionReader) Close() error { return nil } -func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCid cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { // Get all deals containing this piece var deals []struct { @@ -150,7 +150,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec mpd.sp_id = sm.sp_id AND mpd.sector_num = sm.sector_num WHERE - mpd.piece_cid = $1;`, pieceCid.String()) + mpd.piece_cid = $1 AND mpd.piece_length = $2`, pieceCid.String(), pieceSize) if err != nil { return nil, 0, fmt.Errorf("getting piece deals: %w", err) } @@ -183,7 +183,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec return nil, 0, merr } -func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCid cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { // Query parked_pieces and parked_piece_refs in one go var pieceData []struct { ID int64 `db:"id"` @@ -197,9 +197,9 @@ func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, p FROM parked_pieces pp WHERE - pp.piece_cid = $1 AND pp.complete = TRUE AND pp.long_term = TRUE + pp.piece_cid = $1 AND pp.piece_padded_size = $2 AND pp.complete = TRUE AND pp.long_term = TRUE LIMIT 1; - `, pieceCid.String()) + `, pieceCid.String(), pieceSize) if err != nil { return nil, 0, fmt.Errorf("failed to query parked_pieces and parked_piece_refs for piece cid %s: %w", pieceCid.String(), err) } @@ -216,7 +216,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, p return reader, abi.UnpaddedPieceSize(pieceData[0].PieceRawSize), nil } -func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { cacheKey := pieceCid.String() // First check if we have a cached error for this piece @@ -250,16 +250,16 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid readerCtx, readerCtxCancel := context.WithCancel(context.Background()) defer close(r.ready) - reader, size, err := cpr.getPieceReaderFromSector(readerCtx, pieceCid) + reader, size, err := cpr.getPieceReaderFromSector(readerCtx, pieceCid, pieceSize) if err != nil { - log.Warnw("failed to get piece reader from sector", "piececid", pieceCid, "err", err) + log.Warnw("failed to get piece reader from sector", "piececid", pieceCid, "piece size", pieceSize, "err", err) serr := err // Try getPieceReaderFromPiecePark - reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCid) + reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCid, pieceSize) if err != nil { - log.Errorw("failed to get piece reader from piece park", "piececid", pieceCid, "err", err) + log.Errorw("failed to get piece reader from piece park", "piececid", pieceCid, "piece size", pieceSize, "err", err) finalErr := fmt.Errorf("failed to get piece reader from sector or piece park: %w, %w", err, serr) diff --git a/lib/commcidv2/commcidv2.go b/lib/commcidv2/commcidv2.go new file mode 100644 index 000000000..fd5b93f1d --- /dev/null +++ b/lib/commcidv2/commcidv2.go @@ -0,0 +1,175 @@ +package commcidv2 + +import ( + "math/bits" + + filabi "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + pool "github.com/libp2p/go-buffer-pool" + "github.com/multiformats/go-multihash" + "github.com/multiformats/go-varint" + "golang.org/x/xerrors" +) + +type CommP struct { + hashType int8 + treeHeight int8 + payloadPadding uint64 + digest []byte +} + +// hardcoded for npw +const ( + nodeSize = 32 + nodeLog2Size = 5 +) + +var mhMeta = map[int8]struct { + treeArity int8 + nodeLog2Size int8 + pCidV1Pref string + pCidV2Pref string +}{ + 1: { + treeArity: 2, + nodeLog2Size: nodeLog2Size, + pCidV1Pref: "\x01" + "\x81\xE2\x03" + "\x92\x20" + "\x20", // + 32 byte digest == total 39 byte cid + pCidV2Pref: "\x01" + "\x55" + "\x91\x20", // + mh varlen + varpad + int8 height + 32 byte digest == total AT LEAST 39 byte cid + }, +} + +func CommPFromPieceInfo(pi filabi.PieceInfo) (CommP, error) { + var cp CommP + if bits.OnesCount64(uint64(pi.Size)) > 1 { + return cp, xerrors.Errorf("malformed PieceInfo: .Size %d not a power of 2", pi.Size) + } + + // hardcoded until we get another commitment type + cp.hashType = 1 + ks := pi.PieceCID.KeyString() + cp.digest = []byte(ks[len(ks)-nodeSize:]) + + cp.treeHeight = 63 - int8(bits.LeadingZeros64(uint64(pi.Size))) - nodeLog2Size + + return cp, nil +} + +func CommPFromPCidV2(c cid.Cid) (CommP, error) { + var cp CommP + + dmh, err := multihash.Decode(c.Hash()) + if err != nil { + return cp, xerrors.Errorf("decoding cid: %w", err) + } + + // hardcoded for now at https://github.com/multiformats/multicodec/pull/331/files#diff-bf5b449ed8c1850371f42808a186b5c5089edd0025700505a6b8f426cd54a6e4R149 + if dmh.Code != 0x1011 { + return cp, xerrors.Errorf("unexpected multihash code %d", dmh.Code) + } + + p, n, err := varint.FromUvarint(dmh.Digest) + if err != nil { + return cp, xerrors.Errorf("decoding varint: %w", err) + } + + cp.hashType = 1 + cp.payloadPadding = p + cp.treeHeight = int8(dmh.Digest[n]) + cp.digest = dmh.Digest[n+1:] + + return cp, nil +} + +func NewSha2CommP(payloadSize uint64, digest []byte) (CommP, error) { + var cp CommP + + // hardcoded for now + if len(digest) != nodeSize { + return cp, xerrors.Errorf("digest size must be 32, got %d", len(digest)) + } + + psz := payloadSize + + // always 4 nodes long + if psz < 127 { + psz = 127 + } + + // fr32 expansion, count 127 blocks, rounded up + boxSize := ((psz + 126) / 127) * 128 + + // hardcoded for now + cp.hashType = 1 + cp.digest = digest + + cp.treeHeight = 63 - int8(bits.LeadingZeros64(boxSize)) - nodeLog2Size + if bits.OnesCount64(boxSize) != 1 { + cp.treeHeight++ + } + cp.payloadPadding = ((1 << (cp.treeHeight - 2)) * 127) - payloadSize + + return cp, nil +} + +func (cp *CommP) PayloadSize() uint64 { + return (1<<(cp.treeHeight-2))*127 - cp.payloadPadding +} + +func (cp *CommP) PieceLog2Size() int8 { + return cp.treeHeight + nodeLog2Size +} + +func (cp *CommP) PieceInfo() filabi.PieceInfo { + return filabi.PieceInfo{ + Size: filabi.PaddedPieceSize(1 << (cp.treeHeight + nodeLog2Size)), + PieceCID: cp.PCidV1(), // for now it won't understand anything else but V1... I think + } +} + +func (cp *CommP) PCidV1() cid.Cid { + pref := mhMeta[cp.hashType].pCidV1Pref + buf := pool.Get(len(pref) + len(cp.digest)) + copy(buf, pref) + copy(buf[len(pref):], cp.digest) + c, err := cid.Cast(buf) + pool.Put(buf) + if err != nil { + panic(err) + } + return c +} + +func (cp *CommP) PCidV2() cid.Cid { + pref := mhMeta[cp.hashType].pCidV2Pref + + ps := varint.UvarintSize(cp.payloadPadding) + + buf := pool.Get(len(pref) + + 1 + // size of the entire mh "payload" won't exceed 127 bytes + ps + + 1 + // the height is an int8 + nodeSize, // digest size, hardcoded for now + ) + + n := copy(buf, pref) + buf[n] = byte(ps + 1 + nodeSize) + n++ + + n += varint.PutUvarint(buf[n:], cp.payloadPadding) + + buf[n] = byte(cp.treeHeight) + n++ + + copy(buf[n:], cp.digest) + + c, err := cid.Cast(buf) + + pool.Put(buf) + if err != nil { + panic(err) + } + + return c +} + +func (cp *CommP) Digest() []byte { return cp.digest } diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index 88c52a303..c3dcaa66b 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -134,11 +134,11 @@ func (i *IndexStore) Start(ctx context.Context) error { return nil } -// AddIndex adds multihash -> piece cid mappings, along with offset and size information for the piece. -func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan chan Record) error { +// AddIndex adds multihash -> piece cid (v2) mappings, along with offset and size information for the piece. +func (i *IndexStore) AddIndex(ctx context.Context, pieceCidv2 cid.Cid, recordsChan chan Record) error { insertPieceBlockOffsetSize := `INSERT INTO PieceBlockOffsetSize (PieceCid, PayloadMultihash, BlockOffset) VALUES (?, ?, ?)` insertPayloadToPieces := `INSERT INTO PayloadToPieces (PayloadMultihash, PieceCid, BlockSize) VALUES (?, ?, ?)` - pieceCidBytes := pieceCid.Bytes() + pieceCidBytes := pieceCidv2.Bytes() var eg errgroup.Group @@ -161,12 +161,12 @@ func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan if !ok { if len(batchPieceBlockOffsetSize.Entries) > 0 { - if err := i.executeBatchWithRetry(ctx, batchPieceBlockOffsetSize, pieceCid); err != nil { + if err := i.executeBatchWithRetry(ctx, batchPieceBlockOffsetSize, pieceCidv2); err != nil { return err } } if len(batchPayloadToPieces.Entries) > 0 { - if err := i.executeBatchWithRetry(ctx, batchPayloadToPieces, pieceCid); err != nil { + if err := i.executeBatchWithRetry(ctx, batchPayloadToPieces, pieceCidv2); err != nil { return err } } @@ -188,13 +188,13 @@ func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan }) if len(batchPieceBlockOffsetSize.Entries) == i.settings.InsertBatchSize { - if err := i.executeBatchWithRetry(ctx, batchPieceBlockOffsetSize, pieceCid); err != nil { + if err := i.executeBatchWithRetry(ctx, batchPieceBlockOffsetSize, pieceCidv2); err != nil { return err } batchPieceBlockOffsetSize = nil } if len(batchPayloadToPieces.Entries) == i.settings.InsertBatchSize { - if err := i.executeBatchWithRetry(ctx, batchPayloadToPieces, pieceCid); err != nil { + if err := i.executeBatchWithRetry(ctx, batchPayloadToPieces, pieceCidv2); err != nil { return err } batchPayloadToPieces = nil @@ -212,7 +212,7 @@ func (i *IndexStore) AddIndex(ctx context.Context, pieceCid cid.Cid, recordsChan } // executeBatchWithRetry executes a batch with retry logic and exponential backoff -func (i *IndexStore) executeBatchWithRetry(ctx context.Context, batch *gocql.Batch, pieceCid cid.Cid) error { +func (i *IndexStore) executeBatchWithRetry(ctx context.Context, batch *gocql.Batch, pieceCidv2 cid.Cid) error { var err error maxRetries := 20 backoff := 20 * time.Second @@ -236,11 +236,11 @@ func (i *IndexStore) executeBatchWithRetry(ctx context.Context, batch *gocql.Bat return ctx.Err() } - log.Warnf("Batch insert attempt %d failed for piece %s: %v", attempt+1, pieceCid, err) + log.Warnf("Batch insert attempt %d failed for piece %s: %v", attempt+1, pieceCidv2, err) // If max retries reached, return error if attempt == maxRetries { - return xerrors.Errorf("execute batch: executing batch insert for piece %s: %w", pieceCid, err) + return xerrors.Errorf("execute batch: executing batch insert for piece %s: %w", pieceCidv2, err) } // Sleep for backoff duration before retrying @@ -262,8 +262,8 @@ func (i *IndexStore) executeBatchWithRetry(ctx context.Context, batch *gocql.Bat // RemoveIndexes removes all multihash -> piece cid mappings, and all // offset information for the piece. -func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { - pieceCidBytes := pieceCid.Bytes() +func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCidv2 cid.Cid) error { + pieceCidBytes := pieceCidv2.Bytes() // First, select all PayloadMultihash for the given PieceCid from PieceBlockOffsetSize selectQry := `SELECT PayloadMultihash FROM PieceBlockOffsetSize WHERE PieceCid = ?` @@ -278,7 +278,7 @@ func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error payloadMultihashes = append(payloadMultihashes, mhCopy) } if err := iter.Close(); err != nil { - return xerrors.Errorf("scanning PayloadMultihash for piece %s: %w", pieceCid, err) + return xerrors.Errorf("scanning PayloadMultihash for piece %s: %w", pieceCidv2, err) } // Prepare batch deletes for PayloadToPieces @@ -294,16 +294,16 @@ func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error }) if len(batch.Entries) >= batchSize || idx == len(payloadMultihashes)-1 { - if err := i.executeBatchWithRetry(ctx, batch, pieceCid); err != nil { - return xerrors.Errorf("executing batch delete for PayloadToPieces for piece %s: %w", pieceCid, err) + if err := i.executeBatchWithRetry(ctx, batch, pieceCidv2); err != nil { + return xerrors.Errorf("executing batch delete for PayloadToPieces for piece %s: %w", pieceCidv2, err) } batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) } } if len(batch.Entries) >= 0 { - if err := i.executeBatchWithRetry(ctx, batch, pieceCid); err != nil { - return xerrors.Errorf("executing batch delete for PayloadToPieces for piece %s: %w", pieceCid, err) + if err := i.executeBatchWithRetry(ctx, batch, pieceCidv2); err != nil { + return xerrors.Errorf("executing batch delete for PayloadToPieces for piece %s: %w", pieceCidv2, err) } } @@ -311,16 +311,16 @@ func (i *IndexStore) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error delPieceBlockOffsetSizeQry := `DELETE FROM PieceBlockOffsetSize WHERE PieceCid = ?` err := i.session.Query(delPieceBlockOffsetSizeQry, pieceCidBytes).WithContext(ctx).Exec() if err != nil { - return xerrors.Errorf("deleting PieceBlockOffsetSize for piece %s: %w", pieceCid, err) + return xerrors.Errorf("deleting PieceBlockOffsetSize for piece %s: %w", pieceCidv2, err) } return nil } -// PieceInfo contains PieceCid and BlockSize +// PieceInfo contains PieceCidV2 and BlockSize type PieceInfo struct { - PieceCid cid.Cid - BlockSize uint64 + PieceCidV2 cid.Cid + BlockSize uint64 } // PiecesContainingMultihash gets all pieces that contain a multihash along with their BlockSize @@ -337,8 +337,8 @@ func (i *IndexStore) PiecesContainingMultihash(ctx context.Context, m multihash. return nil, fmt.Errorf("parsing piece cid: %w", err) } pieces = append(pieces, PieceInfo{ - PieceCid: pcid, - BlockSize: blockSize, + PieceCidV2: pcid, + BlockSize: blockSize, }) } if err := iter.Close(); err != nil { @@ -352,11 +352,11 @@ func (i *IndexStore) PiecesContainingMultihash(ctx context.Context, m multihash. return pieces, nil } -// GetOffset retrieves the offset of a payload in a piece -func (i *IndexStore) GetOffset(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (uint64, error) { +// GetOffset retrieves the offset of a payload in a piece(v2) +func (i *IndexStore) GetOffset(ctx context.Context, pieceCidv2 cid.Cid, hash multihash.Multihash) (uint64, error) { var offset uint64 qryOffset := `SELECT BlockOffset FROM PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash = ?` - err := i.session.Query(qryOffset, pieceCid.Bytes(), []byte(hash)).WithContext(ctx).Scan(&offset) + err := i.session.Query(qryOffset, pieceCidv2.Bytes(), []byte(hash)).WithContext(ctx).Scan(&offset) if err != nil { return 0, fmt.Errorf("getting offset: %w", err) } @@ -364,9 +364,9 @@ func (i *IndexStore) GetOffset(ctx context.Context, pieceCid cid.Cid, hash multi return offset, nil } -func (i *IndexStore) GetPieceHashRange(ctx context.Context, piece cid.Cid, start multihash.Multihash, num int64) ([]multihash.Multihash, error) { +func (i *IndexStore) GetPieceHashRange(ctx context.Context, piecev2 cid.Cid, start multihash.Multihash, num int64) ([]multihash.Multihash, error) { qry := "SELECT PayloadMultihash FROM PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash >= ? ORDER BY PayloadMultihash ASC LIMIT ?" - iter := i.session.Query(qry, piece.Bytes(), []byte(start), num).WithContext(ctx).Iter() + iter := i.session.Query(qry, piecev2.Bytes(), []byte(start), num).WithContext(ctx).Iter() var hashes []multihash.Multihash var r []byte @@ -378,7 +378,7 @@ func (i *IndexStore) GetPieceHashRange(ctx context.Context, piece cid.Cid, start r = make([]byte, 0, 36) } if err := iter.Close(); err != nil { - return nil, xerrors.Errorf("iterating piece hash range (P:0x%02x, H:0x%02x, n:%d): %w", piece.Bytes(), []byte(start), num, err) + return nil, xerrors.Errorf("iterating piece hash range (P:0x%02x, H:0x%02x, n:%d): %w", piecev2.Bytes(), []byte(start), num, err) } if len(hashes) != int(num) { return nil, xerrors.Errorf("expected %d hashes, got %d (possibly missing indexes)", num, len(hashes)) @@ -387,9 +387,9 @@ func (i *IndexStore) GetPieceHashRange(ctx context.Context, piece cid.Cid, start return hashes, nil } -func (i *IndexStore) CheckHasPiece(ctx context.Context, piece cid.Cid) (bool, error) { +func (i *IndexStore) CheckHasPiece(ctx context.Context, piecev2 cid.Cid) (bool, error) { qry := "SELECT PayloadMultihash FROM PieceBlockOffsetSize WHERE PieceCid = ? AND PayloadMultihash >= ? ORDER BY PayloadMultihash ASC LIMIT ?" - iter := i.session.Query(qry, piece.Bytes(), []byte{0}, 1).WithContext(ctx).Iter() + iter := i.session.Query(qry, piecev2.Bytes(), []byte{0}, 1).WithContext(ctx).Iter() var hashes []multihash.Multihash var r []byte @@ -401,7 +401,7 @@ func (i *IndexStore) CheckHasPiece(ctx context.Context, piece cid.Cid) (bool, er r = make([]byte, 0, 36) } if err := iter.Close(); err != nil { - return false, xerrors.Errorf("iterating piece hash range (P:0x%02x, n:%d): %w", piece.Bytes(), len(hashes), err) + return false, xerrors.Errorf("iterating piece hash range (P:0x%02x, n:%d): %w", piecev2.Bytes(), len(hashes), err) } return len(hashes) > 0, nil diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index 24a6c0693..af776f302 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -111,10 +111,10 @@ func TestNewIndexStore(t *testing.T) { pcids, err := idxStore.PiecesContainingMultihash(ctx, m) require.NoError(t, err) require.Len(t, pcids, 1) - require.Equal(t, pcids[0].PieceCid.String(), commp.PieceCID.String()) + require.Equal(t, pcids[0].PieceCidV2.String(), commp.PieceCID.String()) // Remove all indexes from the store - err = idxStore.RemoveIndexes(ctx, pcids[0].PieceCid) + err = idxStore.RemoveIndexes(ctx, pcids[0].PieceCidV2) require.NoError(t, err) // Drop the tables diff --git a/market/ipni/chunker/serve-chunker.go b/market/ipni/chunker/serve-chunker.go index fb775f28d..a1dd60fed 100644 --- a/market/ipni/chunker/serve-chunker.go +++ b/market/ipni/chunker/serve-chunker.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/curio/lib/storiface" @@ -49,7 +50,7 @@ type ServeChunker struct { entryCache *lru.Cache[cid.Cid, *promise.Promise[result.Result[ipniEntry]]] - // small cache keeping track of which piece CIDs shouldn't be skipped. Entries expire after NoSkipCacheTTL + // small cache keeping track of which piece CIDs (v2) shouldn't be skipped. Entries expire after NoSkipCacheTTL noSkipCache *lru.Cache[cid.Cid, time.Time] } @@ -108,7 +109,7 @@ func (p *ServeChunker) getEntry(rctx context.Context, block cid.Cid, speculated if v.Error == nil { prevChunk = v.Value.Prev return v.Value.Data, nil - } else if v.Error == ErrNotFound { + } else if errors.Is(v.Error, ErrNotFound) { log.Errorw("Cached promise skip", "block", block, "prev", prevChunk, "err", err) return v.Value.Data, v.Error } @@ -129,8 +130,8 @@ func (p *ServeChunker) getEntry(rctx context.Context, block cid.Cid, speculated ctx := context.Background() type ipniChunk struct { - PieceCID string `db:"piece_cid"` - FromCar bool `db:"from_car"` + PieceCIDv2 string `db:"piece_cid"` + FromCar bool `db:"from_car"` FirstCID *string `db:"first_cid"` StartOffset *int64 `db:"start_offset"` @@ -168,12 +169,12 @@ func (p *ServeChunker) getEntry(rctx context.Context, block cid.Cid, speculated } chunk := ipniChunks[0] - pieceCid, err := cid.Parse(chunk.PieceCID) + pieceCidv2, err := cid.Parse(chunk.PieceCIDv2) if err != nil { return nil, xerrors.Errorf("parsing piece CID: %w", err) } - if leave, ok := p.noSkipCache.Get(pieceCid); !ok || time.Now().After(leave) { + if leave, ok := p.noSkipCache.Get(pieceCidv2); !ok || time.Now().After(leave) { skip, err := p.checkIsEntrySkip(ctx, block) if err != nil { return nil, xerrors.Errorf("checking entry skipped for block %s: %w", block, err) @@ -184,7 +185,7 @@ func (p *ServeChunker) getEntry(rctx context.Context, block cid.Cid, speculated } } - p.noSkipCache.Add(pieceCid, time.Now().Add(NoSkipCacheTTL)) + p.noSkipCache.Add(pieceCidv2, time.Now().Add(NoSkipCacheTTL)) var next ipld.Link if chunk.PrevCID != nil { @@ -208,23 +209,30 @@ func (p *ServeChunker) getEntry(rctx context.Context, block cid.Cid, speculated firstHash := multihash.Multihash(cb) - return p.reconstructChunkFromDB(ctx, block, pieceCid, firstHash, next, chunk.NumBlocks, speculated) + return p.reconstructChunkFromDB(ctx, block, pieceCidv2, firstHash, next, chunk.NumBlocks, speculated) } - return p.reconstructChunkFromCar(ctx, block, pieceCid, *chunk.StartOffset, next, chunk.NumBlocks, speculated) + return p.reconstructChunkFromCar(ctx, block, pieceCidv2, *chunk.StartOffset, next, chunk.NumBlocks, speculated) } // reconstructChunkFromCar reconstructs a chunk from a car file. -func (p *ServeChunker) reconstructChunkFromCar(ctx context.Context, chunk, piece cid.Cid, startOff int64, next ipld.Link, numBlocks int64, speculate bool) ([]byte, error) { +func (p *ServeChunker) reconstructChunkFromCar(ctx context.Context, chunk, piecev2 cid.Cid, startOff int64, next ipld.Link, numBlocks int64, speculate bool) ([]byte, error) { start := time.Now() - reader, _, err := p.cpr.GetSharedPieceReader(ctx, piece) + commp, err := commcidv2.CommPFromPCidV2(piecev2) + if err != nil { + return nil, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + + pi := commp.PieceInfo() + + reader, _, err := p.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) defer func(reader storiface.Reader) { _ = reader.Close() }(reader) if err != nil { - return nil, xerrors.Errorf("failed to read piece %s for ipni chunk %s reconstruction: %w", piece, chunk, err) + return nil, xerrors.Errorf("failed to read piece %s of size %d for ipni chunk %s reconstruction: %w", pi.PieceCID, pi.Size, chunk, err) } _, err = reader.Seek(startOff, io.SeekStart) @@ -274,16 +282,23 @@ func (p *ServeChunker) reconstructChunkFromCar(ctx context.Context, chunk, piece return nil, xerrors.Errorf("encoding chunk node: %w", err) } - log.Infow("Reconstructing chunk from car", "chunk", chunk, "piece", piece, "startOffset", startOff, "numBlocks", numBlocks, "speculated", speculate, "readMiB", float64(curOff-startOff)/1024/1024, "recomputeTime", time.Since(read), "totalTime", time.Since(start), "ents/s", float64(numBlocks)/time.Since(start).Seconds(), "MiB/s", float64(curOff-startOff)/1024/1024/time.Since(start).Seconds()) + log.Infow("Reconstructing chunk from car", "chunk", chunk, "piece", pi.PieceCID, "size", pi.Size, "startOffset", startOff, "numBlocks", numBlocks, "speculated", speculate, "readMiB", float64(curOff-startOff)/1024/1024, "recomputeTime", time.Since(read), "totalTime", time.Since(start), "ents/s", float64(numBlocks)/time.Since(start).Seconds(), "MiB/s", float64(curOff-startOff)/1024/1024/time.Since(start).Seconds()) return b.Bytes(), nil } // ReconstructChunkFromDB reconstructs a chunk from the database. -func (p *ServeChunker) reconstructChunkFromDB(ctx context.Context, chunk, piece cid.Cid, firstHash multihash.Multihash, next ipld.Link, numBlocks int64, speculate bool) ([]byte, error) { +func (p *ServeChunker) reconstructChunkFromDB(ctx context.Context, chunk, piecev2 cid.Cid, firstHash multihash.Multihash, next ipld.Link, numBlocks int64, speculate bool) ([]byte, error) { start := time.Now() - mhs, err := p.indexStore.GetPieceHashRange(ctx, piece, firstHash, numBlocks) + commp, err := commcidv2.CommPFromPCidV2(piecev2) + if err != nil { + return nil, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + + pi := commp.PieceInfo() + + mhs, err := p.indexStore.GetPieceHashRange(ctx, piecev2, firstHash, numBlocks) if err != nil { return nil, xerrors.Errorf("getting piece hash range: %w", err) } @@ -315,7 +330,7 @@ func (p *ServeChunker) reconstructChunkFromDB(ctx context.Context, chunk, piece return nil, err } - log.Infow("Reconstructing chunk from DB", "chunk", chunk, "piece", piece, "firstHash", firstHash, "numBlocks", numBlocks, "speculated", speculate, "totalTime", time.Since(start), "ents/s", float64(numBlocks)/time.Since(start).Seconds()) + log.Infow("Reconstructing chunk from DB", "chunk", chunk, "piece", pi.PieceCID, "size", pi.Size, "firstHash", firstHash, "numBlocks", numBlocks, "speculated", speculate, "totalTime", time.Since(start), "ents/s", float64(numBlocks)/time.Since(start).Seconds()) return b.Bytes(), nil } diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index 9de4fe2bb..bee914785 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -33,40 +33,40 @@ type DDOV1 struct { // Client represents the address of the deal client Client address.Address `json:"client"` - // Actor able to with AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs - PieceManager address.Address `json:"piecemanager"` + // Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs + PieceManager address.Address `json:"piece_manager"` // Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. // It must be at least 518400 Duration abi.ChainEpoch `json:"duration"` // AllocationId represents an aggregated allocation identifier for the deal. - AllocationId *verifreg.AllocationId `json:"aggregatedallocationid"` + AllocationId *verifreg.AllocationId `json:"allocation_id"` // ContractAddress specifies the address of the contract governing the deal - ContractAddress string `json:"contractaddress"` + ContractAddress string `json:"contract_address"` - // ContractDealIDMethod specifies the method name to retrieve the deal ID for a contract - ContractDealIDMethod string `json:"contractdealidmethod"` + // ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract + ContractVerifyMethod string `json:"contract_verify_method"` - // ContractDealIDMethodParams represents encoded parameters for the contract deal ID method if required by the contract - ContractDealIDMethodParams []byte `json:"contractdealidmethodparams"` + // ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract + ContractVerifyMethodParams []byte `json:"contract_verify_method_params"` // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated - NotificationAddress string `json:"notificationaddress"` + NotificationAddress string `json:"notification_address"` // NotificationPayload holds the notification data typically in a serialized byte array format. - NotificationPayload []byte `json:"notificationpayload"` + NotificationPayload []byte `json:"notification_payload"` // Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval Indexing bool `json:"indexing"` // AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). - AnnounceToIPNI bool `json:"announcetoinpni"` + AnnounceToIPNI bool `json:"announce_to_ipni"` } -func (d *DDOV1) Validate(dbProducts []dbProduct) (ErrorCode, error) { - code, err := d.IsEnabled(dbProducts) +func (d *DDOV1) Validate(db *harmonydb.DB) (ErrorCode, error) { + code, err := IsProductEnabled(db, d.ProductName()) if err != nil { return code, err } @@ -103,12 +103,16 @@ func (d *DDOV1) Validate(dbProducts []dbProduct) (ErrorCode, error) { return ErrProductValidationFailed, xerrors.Errorf("contract address must start with 0x") } - if d.ContractDealIDMethodParams == nil { - return ErrProductValidationFailed, xerrors.Errorf("contract deal id method params is not set") + if d.ContractVerifyMethodParams == nil { + return ErrProductValidationFailed, xerrors.Errorf("contract verify method params is not set") } - if d.ContractDealIDMethod == "" { - return ErrProductValidationFailed, xerrors.Errorf("contract deal id method is not set") + if d.ContractVerifyMethod == "" { + return ErrProductValidationFailed, xerrors.Errorf("contract verify method is not set") + } + + if !d.Indexing && d.AnnounceToIPNI { + return ErrProductValidationFailed, xerrors.Errorf("deal cannot be announced to IPNI without indexing") } return Ok, nil @@ -132,9 +136,9 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. to := common.HexToAddress(d.ContractAddress) // Get the method - method, exists := parsedABI.Methods[d.ContractDealIDMethod] + method, exists := parsedABI.Methods[d.ContractVerifyMethod] if !exists { - return "", http.StatusInternalServerError, fmt.Errorf("method %s not found in ABI", d.ContractDealIDMethod) + return "", http.StatusInternalServerError, fmt.Errorf("method %s not found in ABI", d.ContractVerifyMethod) } // Enforce method must take exactly one `bytes` parameter @@ -143,7 +147,7 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. } // ABI-encode method call with input - callData, err := parsedABI.Pack(method.Name, d.ContractDealIDMethodParams) + callData, err := parsedABI.Pack(method.Name, d.ContractVerifyMethod) if err != nil { return "", http.StatusInternalServerError, fmt.Errorf("failed to encode call data: %w", err) } @@ -176,16 +180,3 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. func (d *DDOV1) ProductName() ProductName { return ProductNameDDOV1 } - -func (d *DDOV1) IsEnabled(dbProducts []dbProduct) (ErrorCode, error) { - name := string(d.ProductName()) - for _, p := range dbProducts { - if p.Name == name { - if p.Enabled { - return Ok, nil - } - return ErrProductNotEnabled, xerrors.Errorf("product %s is not enabled on the provider", name) - } - } - return ErrUnsupportedProduct, xerrors.Errorf("product %s is not supported on the provider", name) -} diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 7323430a3..f06f22e31 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -75,30 +75,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { // Validate the DataSource TODO: Add error code to validate - var dbProducts []dbProduct - err := m.db.Select(context.Background(), &dbProducts, `SELECT name, enabled FROM products`) - if err != nil { - log.Errorw("error getting products from DB", "error", err) - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, - } - } - - var dbDataSources []dbDataSource - err = m.db.Select(context.Background(), &dbDataSources, `SELECT name, enabled FROM data_sources`) - if err != nil { - log.Errorw("error getting data sources from DB", "error", err) - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, - } - } - - vdata := &productAndDataSource{ - Products: dbProducts, - Data: dbDataSources, - } - - code, err := deal.Validate(vdata) + code, err := deal.Validate(m.db) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ @@ -199,6 +176,15 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } + if deal.Data.Format.Raw != nil { + if deal.Products.DDOV1.Indexing { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Raw bytes deal cannot be indexed", + }, nil + } + } + if deal.Products.DDOV1.AllocationId != nil { if deal.Data.Size < abi.PaddedPieceSize(verifreg.MinimumVerifiedAllocationSize) { return &ProviderDealRejectionInfo{ diff --git a/market/mk20/types.go b/market/mk20/types.go index fc8115778..a9f615c9d 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -3,11 +3,9 @@ package mk20 import ( "net/http" + "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" "github.com/oklog/ulid" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-state-types/abi" ) // Deal represents a structure defining the details and components of a specific deal in the system. @@ -25,14 +23,14 @@ type Deal struct { type Products struct { // DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) - DDOV1 *DDOV1 `json:"ddov1"` + DDOV1 *DDOV1 `json:"ddo_v1"` } // DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. type DataSource struct { // PieceCID represents the unique identifier for a piece of data, stored as a CID object. - PieceCID cid.Cid `json:"piececid"` + PieceCID cid.Cid `json:"piece_cid"` // Size represents the size of the padded piece in the data source. Size abi.PaddedPieceSize `json:"size"` @@ -41,16 +39,16 @@ type DataSource struct { Format PieceDataFormat `json:"format"` // SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. - SourceHTTP *DataSourceHTTP `json:"sourcehttp"` + SourceHTTP *DataSourceHTTP `json:"source_http"` // SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. - SourceAggregate *DataSourceAggregate `json:"sourceaggregate"` + SourceAggregate *DataSourceAggregate `json:"source_aggregate"` // SourceOffline defines the data source for offline pieces, including raw size information. - SourceOffline *DataSourceOffline `json:"sourceoffline"` + SourceOffline *DataSourceOffline `json:"source_offline"` // SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import - SourceHttpPut *DataSourceHttpPut `json:"sourcehttpput"` + SourceHttpPut *DataSourceHttpPut `json:"source_httpput"` // SourceStorageProvider -> sp IDs/ipni, pieceCids } @@ -68,11 +66,8 @@ type PieceDataFormat struct { Raw *FormatBytes `json:"raw"` } -// FormatCar represents the CAR (Content Addressable aRchive) format with version metadata for piece data serialization. -type FormatCar struct { - // Version specifies the version of the CAR format used for piece data serialization. - Version uint64 `json:"version"` -} +// FormatCar represents the CAR (Content Addressable archive) format for piece data serialization. +type FormatCar struct{} // FormatAggregate represents the aggregated format for piece data, identified by its type. type FormatAggregate struct { @@ -91,23 +86,7 @@ type FormatBytes struct{} // DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. type DataSourceOffline struct { // RawSize specifies the raw size of the data in bytes. - RawSize uint64 `json:"rawsize"` -} - -func (dso *DataSourceOffline) Name() DataSourceName { - return DataSourceNameOffline -} - -func (dso *DataSourceOffline) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { - name := string(dso.Name()) - for _, p := range dbDataSources { - if p.Name == name { - if p.Enabled { - return Ok, nil - } - } - } - return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) + RawSize uint64 `json:"raw_size"` } // DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. @@ -115,22 +94,6 @@ type DataSourceAggregate struct { Pieces []DataSource `json:"pieces"` } -func (dsa *DataSourceAggregate) Name() DataSourceName { - return DataSourceNameAggregate -} - -func (dsa *DataSourceAggregate) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { - name := string(dsa.Name()) - for _, p := range dbDataSources { - if p.Name == name { - if p.Enabled { - return Ok, nil - } - } - } - return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) -} - // DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. type DataSourceHTTP struct { @@ -141,22 +104,6 @@ type DataSourceHTTP struct { URLs []HttpUrl `json:"urls"` } -func (dsh *DataSourceHTTP) Name() DataSourceName { - return DataSourceNameHTTP -} - -func (dsh *DataSourceHTTP) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { - name := string(dsh.Name()) - for _, p := range dbDataSources { - if p.Name == name { - if p.Enabled { - return Ok, nil - } - } - } - return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) -} - // HttpUrl represents an HTTP endpoint configuration for fetching piece data. type HttpUrl struct { @@ -164,7 +111,7 @@ type HttpUrl struct { URL string `json:"url"` // HTTPHeaders represents the HTTP headers associated with the URL. - HTTPHeaders http.Header `json:"httpheaders"` + Headers http.Header `json:"headers"` // Priority indicates the order preference for using the URL in requests, with lower values having higher priority. Priority uint64 `json:"priority"` @@ -176,23 +123,7 @@ type HttpUrl struct { // DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted. type DataSourceHttpPut struct { // RawSize specifies the raw size of the data in bytes. - RawSize uint64 `json:"rawsize"` -} - -func (dsh *DataSourceHttpPut) Name() DataSourceName { - return DataSourceNamePut -} - -func (dsh *DataSourceHttpPut) IsEnabled(dbDataSources []dbDataSource) (ErrorCode, error) { - name := string(dsh.Name()) - for _, p := range dbDataSources { - if p.Name == name { - if p.Enabled { - return Ok, nil - } - } - } - return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) + RawSize uint64 `json:"raw_size"` } // AggregateType represents an unsigned integer used to define the type of aggregation for data pieces in the system. @@ -253,8 +184,8 @@ const ( type ProductName string const ( - // ProductNameDDOV1 represents the identifier for the "ddov1" product used in contract operations and validations. - ProductNameDDOV1 ProductName = "ddov1" + // ProductNameDDOV1 represents the identifier for the "ddo_v1" product used in contract operations and validations. + ProductNameDDOV1 ProductName = "ddo_v1" ) type DataSourceName string @@ -263,7 +194,7 @@ const ( DataSourceNameHTTP DataSourceName = "http" DataSourceNameAggregate DataSourceName = "aggregate" DataSourceNameOffline DataSourceName = "offline" - DataSourceNameStorageProvider DataSourceName = "storageprovider" + DataSourceNameStorageProvider DataSourceName = "storage_provider" DataSourceNamePDP DataSourceName = "pdp" DataSourceNamePut DataSourceName = "put" ) diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go deleted file mode 100644 index adf2d4599..000000000 --- a/market/mk20/types_test.go +++ /dev/null @@ -1,331 +0,0 @@ -package mk20 - -import ( - "testing" - - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-padreader" - "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" -) - -var ( - testCID, _ = cid.Parse("baga6ea4seaqnnlsm75qhc4h76ts6bytfdxf6epjgqlhozjtuony4fwlui2xfuhq") - raw2MiB = uint64(2 << 20) // unā€‘padded 2MiB - padded2MiB = padreader.PaddedSize(raw2MiB).Padded() -) - -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -// helpers to create *valid* structs that individual testā€‘cases mutate -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - -func validDBDataSources() []dbDataSource { - return []dbDataSource{ - {Name: "http", Enabled: true}, - {Name: "offline", Enabled: true}, - {Name: "aggregate", Enabled: true}, - {Name: "put", Enabled: true}, - } -} - -func validDBProducts() []dbProduct { return []dbProduct{{Name: "ddov1", Enabled: true}} } - -func validDataSource() DataSource { - return DataSource{ - PieceCID: testCID, - Size: padded2MiB, - Format: PieceDataFormat{ - Car: &FormatCar{Version: 1}, - }, - SourceHTTP: &DataSourceHTTP{ - RawSize: raw2MiB, - URLs: []HttpUrl{{URL: "https://example.com/file.car"}}, - }, - } -} - -func validDDOV1() *DDOV1 { - sp, _ := address.NewFromString("f01234") - cl, _ := address.NewFromString("f05678") - pm, _ := address.NewFromString("f09999") - - return &DDOV1{ - Provider: sp, - Client: cl, - PieceManager: pm, - Duration: 518400, - ContractAddress: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - ContractDealIDMethod: "dealID", - ContractDealIDMethodParams: []byte{0x01}, - } -} - -func validDeal(t *testing.T) Deal { - id, err := NewULID() - require.NoError(t, err) - return Deal{ - Identifier: id, - Data: validDataSource(), - Products: Products{DDOV1: validDDOV1()}, - } -} - -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -// 1. Products.Validate + DDOV1.Validate -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - -func TestValidate_DDOV1(t *testing.T) { - base := *validDDOV1() // copy - tests := []struct { - name string - prod []dbProduct - mutate func(*DDOV1) - wantCode ErrorCode - }{ - // enabled / disabled / unsupported - {"no products on provider", - nil, - func(d *DDOV1) {}, - ErrUnsupportedProduct}, - {"product disabled", - []dbProduct{{Name: "ddov1", Enabled: false}}, - func(d *DDOV1) {}, - ErrProductNotEnabled}, - {"product unsupported", - []dbProduct{{Name: "other", Enabled: true}}, - func(d *DDOV1) {}, - ErrUnsupportedProduct}, - - // fieldā€‘level failures - {"provider undef", validDBProducts(), - func(d *DDOV1) { d.Provider = address.Undef }, - ErrProductValidationFailed}, - {"client undef", validDBProducts(), - func(d *DDOV1) { d.Client = address.Undef }, - ErrProductValidationFailed}, - {"pieceā€‘manager undef", validDBProducts(), - func(d *DDOV1) { d.PieceManager = address.Undef }, - ErrProductValidationFailed}, - {"allocation id == NoAllocationID", validDBProducts(), - func(d *DDOV1) { - na := verifreg.NoAllocationID - d.AllocationId = &na - }, - ErrProductValidationFailed}, - {"duration too short", validDBProducts(), - func(d *DDOV1) { d.Duration = 10 }, - ErrDurationTooShort}, - {"contract address empty", validDBProducts(), - func(d *DDOV1) { d.ContractAddress = "" }, - ErrProductValidationFailed}, - {"contract address no 0x", validDBProducts(), - func(d *DDOV1) { d.ContractAddress = "abc" }, - ErrProductValidationFailed}, - {"contract params nil", validDBProducts(), - func(d *DDOV1) { d.ContractDealIDMethodParams = nil }, - ErrProductValidationFailed}, - {"contract method empty", validDBProducts(), - func(d *DDOV1) { d.ContractDealIDMethod = "" }, - ErrProductValidationFailed}, - - // happy path - {"happy path", validDBProducts(), - func(d *DDOV1) {}, - Ok}, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - d := base - tc.mutate(&d) - code, _ := d.Validate(tc.prod) - require.Equal(t, tc.wantCode, code) - }) - } -} - -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -// 2. DataSource.Validate (all branches) -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - -func TestValidate_DataSource(t *testing.T) { - baseDB := validDBDataSources() - tests := []struct { - name string - mutateDS func(*DataSource) - mutateDBSrc func([]dbDataSource) []dbDataSource - wantCode ErrorCode - }{ - // providerā€‘level enable / disable checks - {"no data sources enabled", - func(ds *DataSource) {}, func(_ []dbDataSource) []dbDataSource { return nil }, - ErrUnsupportedDataSource}, - {"http disabled", - func(ds *DataSource) {}, - func(src []dbDataSource) []dbDataSource { src[0].Enabled = false; return src }, - ErrUnsupportedDataSource}, - - // topā€‘level sanity - {"undefined CID", - func(ds *DataSource) { ds.PieceCID = cid.Undef }, - nil, ErrBadProposal}, - {"size zero", - func(ds *DataSource) { ds.Size = 0 }, - nil, ErrBadProposal}, - {"no source defined", - func(ds *DataSource) { - ds.SourceHTTP = nil - }, nil, ErrBadProposal}, - {"multiple sources defined", - func(ds *DataSource) { - ds.SourceOffline = &DataSourceOffline{RawSize: raw2MiB} - ds.SourceHttpPut = &DataSourceHttpPut{RawSize: raw2MiB} - ds.SourceAggregate = &DataSourceAggregate{Pieces: []DataSource{}} - }, nil, ErrBadProposal}, - - // format combinations - {"no format", - func(ds *DataSource) { ds.Format = PieceDataFormat{} }, - nil, ErrBadProposal}, - {"multiple formats", - func(ds *DataSource) { - ds.Format.Raw = &FormatBytes{} - }, nil, ErrBadProposal}, - {"car version unsupported", - func(ds *DataSource) { ds.Format.Car.Version = 3 }, - nil, ErrMalformedDataSource}, - - // HTTP source specific - {"http rawsize zero", - func(ds *DataSource) { ds.SourceHTTP.RawSize = 0 }, - nil, ErrMalformedDataSource}, - {"http urls empty", - func(ds *DataSource) { ds.SourceHTTP.URLs = nil }, - nil, ErrMalformedDataSource}, - {"http url invalid", - func(ds *DataSource) { ds.SourceHTTP.URLs[0].URL = "::::" }, - nil, ErrMalformedDataSource}, - - // Offline source - {"offline source disabled", - func(ds *DataSource) { - ds.SourceHTTP = nil - ds.SourceOffline = &DataSourceOffline{RawSize: raw2MiB} - }, - func(src []dbDataSource) []dbDataSource { src[1].Enabled = false; return src }, - ErrUnsupportedDataSource}, - {"offline rawsize zero", - func(ds *DataSource) { - ds.SourceHTTP = nil - ds.SourceOffline = &DataSourceOffline{RawSize: 0} - }, nil, ErrMalformedDataSource}, - - // HttpPut source - {"put source disabled", - func(ds *DataSource) { - ds.SourceHTTP = nil - ds.SourceHttpPut = &DataSourceHttpPut{RawSize: raw2MiB} - }, - func(src []dbDataSource) []dbDataSource { src[3].Enabled = false; return src }, - ErrUnsupportedDataSource}, - {"put rawsize zero", - func(ds *DataSource) { - ds.SourceHTTP = nil - ds.SourceHttpPut = &DataSourceHttpPut{RawSize: 0} - }, nil, ErrMalformedDataSource}, - - // Size mismatch on final check - {"declared size mismatch", - func(ds *DataSource) { ds.Size *= 2 }, - nil, ErrBadProposal}, - - // happy path - {"happy path", - func(ds *DataSource) {}, - nil, Ok}, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - ds := validDataSource() - tc.mutateDS(&ds) - - db := baseDB - if tc.mutateDBSrc != nil { - db = tc.mutateDBSrc(append([]dbDataSource(nil), baseDB...)) - } - code, _ := ds.Validate(db) - require.Equal(t, tc.wantCode, code) - }) - } -} - -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -// 3. Deal.Validate (composition of the two) -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - -func TestValidate_Deal(t *testing.T) { - tests := []struct { - name string - mutate func(*Deal, *productAndDataSource) - wantCode ErrorCode - }{ - {"happy path", - func(d *Deal, _ *productAndDataSource) {}, - Ok}, - - // propagate product failure - {"product failure bubbles", - func(d *Deal, pad *productAndDataSource) { - pad.Products[0].Enabled = false // DDOV1 disabled - }, ErrProductNotEnabled}, - - // propagate data source failure - {"data source failure bubbles", - func(d *Deal, _ *productAndDataSource) { - d.Data.PieceCID = cid.Undef - }, ErrBadProposal}, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - deal := validDeal(t) - pad := &productAndDataSource{ - Products: append([]dbProduct(nil), validDBProducts()...), - Data: append([]dbDataSource(nil), validDBDataSources()...), - } - tc.mutate(&deal, pad) - code, _ := deal.Validate(pad) - require.Equal(t, tc.wantCode, code) - }) - } -} - -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -// 4. quick sanity that URL parsing in Validate works on aggregate subā€‘pieces -// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ - -func TestValidate_Aggregate_SubPieceChecks(t *testing.T) { - // base structure: an aggregate of one valid HTTP piece - sub := validDataSource() - agg := validDataSource() - agg.Format = PieceDataFormat{ - Aggregate: &FormatAggregate{ - Type: AggregateTypeV1, - Sub: nil, - }, - } - agg.SourceHTTP = nil - agg.SourceAggregate = &DataSourceAggregate{Pieces: []DataSource{sub}} - - // (size will mismatch ā€“ test expects that specific error branch) - agg.Size = padded2MiB * 8 - - code, _ := agg.Validate(validDBDataSources()) - require.Equal(t, ErrBadProposal, code) -} diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 729b02459..b59856599 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -3,14 +3,19 @@ package mk20 import ( "context" "crypto/rand" + "database/sql" "encoding/json" + "errors" "fmt" "math/bits" + "net/http" "net/url" "time" + "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-data-segment/datasegment" @@ -20,29 +25,16 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" ) -type dbDataSource struct { - Name string `db:"name"` - Enabled bool `db:"enabled"` -} - -type productAndDataSource struct { - Products []dbProduct - Data []dbDataSource -} - -func (d *Deal) Validate(pad *productAndDataSource) (ErrorCode, error) { - code, err := d.Products.Validate(pad.Products) +func (d *Deal) Validate(db *harmonydb.DB) (ErrorCode, error) { + code, err := d.Products.Validate(db) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) } - return d.Data.Validate(pad.Data) + return d.Data.Validate(db) } -func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { - if len(dbDataSources) == 0 { - return ErrUnsupportedDataSource, xerrors.Errorf("no data sources enabled on the provider") - } +func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { if !d.PieceCID.Defined() { return ErrBadProposal, xerrors.Errorf("piece cid is not defined") @@ -64,9 +56,6 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { if d.Format.Car != nil { fcar = true - if d.Format.Car.Version != 1 && d.Format.Car.Version != 2 { - return ErrMalformedDataSource, xerrors.Errorf("car version not supported") - } } if d.Format.Aggregate != nil { @@ -77,7 +66,7 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { } if d.SourceAggregate != nil { - code, err := d.SourceAggregate.IsEnabled(dbDataSources) + code, err := IsDataSourceEnabled(db, d.SourceAggregate.Name()) if err != nil { return code, err } @@ -99,9 +88,6 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { if p.Format.Car != nil { ifcar = true - if p.Format.Car.Version != 1 && p.Format.Car.Version != 2 { - return ErrMalformedDataSource, xerrors.Errorf("car version not supported") - } } if p.Format.Aggregate != nil { @@ -156,6 +142,10 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { } } + } else { + if len(d.Format.Aggregate.Sub) == 0 { + return ErrMalformedDataSource, xerrors.Errorf("no sub pieces defined under aggregate") + } } } @@ -172,7 +162,7 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { } if d.SourceHTTP != nil { - code, err := d.SourceHTTP.IsEnabled(dbDataSources) + code, err := IsDataSourceEnabled(db, d.SourceHTTP.Name()) if err != nil { return code, err } @@ -194,7 +184,7 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { } if d.SourceOffline != nil { - code, err := d.SourceOffline.IsEnabled(dbDataSources) + code, err := IsDataSourceEnabled(db, d.SourceOffline.Name()) if err != nil { return code, err } @@ -205,7 +195,7 @@ func (d DataSource) Validate(dbDataSources []dbDataSource) (ErrorCode, error) { } if d.SourceHttpPut != nil { - code, err := d.SourceHttpPut.IsEnabled(dbDataSources) + code, err := IsDataSourceEnabled(db, d.SourceHttpPut.Name()) if err != nil { return code, err } @@ -271,42 +261,30 @@ func (d DataSource) RawSize() (uint64, error) { return 0, xerrors.Errorf("no source defined") } -type dbProduct struct { - Name string `db:"name"` - Enabled bool `db:"enabled"` -} - -func (d Products) Validate(dbProducts []dbProduct) (ErrorCode, error) { - if len(dbProducts) == 0 { - return ErrProductNotEnabled, xerrors.Errorf("no products enabled on the provider") - } - +func (d Products) Validate(db *harmonydb.DB) (ErrorCode, error) { if d.DDOV1 == nil { return ErrBadProposal, xerrors.Errorf("no products") } - return d.DDOV1.Validate(dbProducts) + return d.DDOV1.Validate(db) } type DBDeal struct { Identifier string `db:"id"` + SpID int64 `db:"sp_id"` PieceCID string `db:"piece_cid"` Size int64 `db:"size"` Format json.RawMessage `db:"format"` SourceHTTP json.RawMessage `db:"source_http"` SourceAggregate json.RawMessage `db:"source_aggregate"` SourceOffline json.RawMessage `db:"source_offline"` - DDOv1 json.RawMessage `db:"ddov1"` + SourceHttpPut json.RawMessage `db:"source_http_put"` + DDOv1 json.RawMessage `db:"ddo_v1"` + Error sql.NullString `db:"error"` } func (d *Deal) ToDBDeal() (*DBDeal, error) { - - // Marshal Format (always present) - formatBytes, err := json.Marshal(d.Data.Format) - if err != nil { - return nil, fmt.Errorf("marshal format: %w", err) - } - + var err error // Marshal SourceHTTP (optional) var sourceHTTPBytes []byte if d.Data.SourceHTTP != nil { @@ -325,6 +303,17 @@ func (d *Deal) ToDBDeal() (*DBDeal, error) { if err != nil { return nil, fmt.Errorf("marshal source_aggregate: %w", err) } + if len(d.Data.SourceAggregate.Pieces) > 0 && len(d.Data.SourceAggregate.Pieces) != len(d.Data.Format.Aggregate.Sub) { + var subPieces []PieceDataFormat + for _, p := range d.Data.SourceAggregate.Pieces { + subPieces = append(subPieces, PieceDataFormat{ + Car: p.Format.Car, + Raw: p.Format.Raw, + Aggregate: p.Format.Aggregate, + }) + } + d.Data.Format.Aggregate.Sub = subPieces + } } else { sourceAggregateBytes = []byte("null") } @@ -340,24 +329,49 @@ func (d *Deal) ToDBDeal() (*DBDeal, error) { sourceOfflineBytes = []byte("null") } + var sourceHttpPutBytes []byte + if d.Data.SourceHttpPut != nil { + sourceHttpPutBytes, err = json.Marshal(d.Data.SourceHttpPut) + if err != nil { + return nil, fmt.Errorf("marshal source_http_put: %w", err) + } + } else { + sourceHttpPutBytes = []byte("null") + } + + // Marshal Format (always present) + formatBytes, err := json.Marshal(d.Data.Format) + if err != nil { + return nil, fmt.Errorf("marshal format: %w", err) + } + + var spid abi.ActorID + var ddov1 []byte if d.Products.DDOV1 != nil { ddov1, err = json.Marshal(d.Products.DDOV1) if err != nil { return nil, fmt.Errorf("marshal ddov1: %w", err) } + spidInt, err := address.IDFromAddress(d.Products.DDOV1.Provider) + if err != nil { + return nil, fmt.Errorf("parse provider address: %w", err) + } + spid = abi.ActorID(spidInt) } else { ddov1 = []byte("null") } return &DBDeal{ Identifier: d.Identifier.String(), + SpID: int64(spid), PieceCID: d.Data.PieceCID.String(), Size: int64(d.Data.Size), Format: formatBytes, SourceHTTP: sourceHTTPBytes, SourceAggregate: sourceAggregateBytes, SourceOffline: sourceOfflineBytes, + SourceHttpPut: sourceHttpPutBytes, DDOv1: ddov1, }, nil } @@ -368,15 +382,17 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } - n, err := tx.Exec(`INSERT INTO deals (id, piece_cid, size, format, source_http, source_aggregate, source_offline, ddov1) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + n, err := tx.Exec(`INSERT INTO deals (id, sp_id, piece_cid, size, format, source_http, source_aggregate, source_offline, source_http_put, ddo_v1) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, dbDeal.Identifier, + dbDeal.SpID, dbDeal.PieceCID, dbDeal.Size, dbDeal.Format, dbDeal.SourceHTTP, dbDeal.SourceAggregate, dbDeal.SourceOffline, + dbDeal.SourceHttpPut, dbDeal.DDOv1) if err != nil { return xerrors.Errorf("insert deal: %w", err) @@ -389,7 +405,7 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal - err := tx.Select(&dbDeal, `SELECT * FROM deals WHERE id = $1`, id.String()) + err := tx.Select(&dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -401,7 +417,7 @@ func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { func DealFromDB(ctx context.Context, db *harmonydb.DB, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal - err := db.Select(ctx, &dbDeal, `SELECT * FROM deals WHERE id = $1`, id.String()) + err := db.Select(ctx, &dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -497,7 +513,7 @@ type DealStatusResponse struct { State DealState `json:"status"` // ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. - ErrorMsg string `json:"errormsg"` + ErrorMsg string `json:"error_msg"` } // DealStatus represents the status of a deal, including the HTTP code and an optional response detailing the deal's state and error message. @@ -543,3 +559,50 @@ type SupportedContracts struct { func NewULID() (ulid.ULID, error) { return ulid.New(ulid.Timestamp(time.Now()), rand.Reader) } + +func (dsh *DataSourceHTTP) Name() DataSourceName { + return DataSourceNameHTTP +} + +func (dso *DataSourceOffline) Name() DataSourceName { + return DataSourceNameOffline +} + +func (dsa *DataSourceAggregate) Name() DataSourceName { + return DataSourceNameAggregate +} + +func (dsh *DataSourceHttpPut) Name() DataSourceName { + return DataSourceNamePut +} + +func IsDataSourceEnabled(db *harmonydb.DB, name DataSourceName) (ErrorCode, error) { + var enabled bool + + err := db.QueryRow(context.Background(), `SELECT enabled FROM market_mk20_data_source WHERE name = $1`, name).Scan(&enabled) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return http.StatusInternalServerError, xerrors.Errorf("data source %s is not enabled", name) + } + } + if !enabled { + return ErrUnsupportedDataSource, xerrors.Errorf("data source %s is not enabled", name) + } + return Ok, nil +} + +func IsProductEnabled(db *harmonydb.DB, name ProductName) (ErrorCode, error) { + var enabled bool + + err := db.QueryRow(context.Background(), `SELECT enabled FROM market_mk20_products WHERE name = $1`, name).Scan(&enabled) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return http.StatusInternalServerError, xerrors.Errorf("data source %s is not enabled", name) + } + return ErrUnsupportedProduct, xerrors.Errorf("product %s is not supported by the provider", name) + } + if !enabled { + return ErrProductNotEnabled, xerrors.Errorf("product %s is not enabled", name) + } + return Ok, nil +} diff --git a/market/retrieval/piecehandler.go b/market/retrieval/piecehandler.go index 931b0eb5b..3449b1b8b 100644 --- a/market/retrieval/piecehandler.go +++ b/market/retrieval/piecehandler.go @@ -13,6 +13,7 @@ import ( "go.opencensus.io/stats" "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/retrieval/remoteblockstore" ) @@ -44,8 +45,18 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { return } + commp, err := commcidv2.CommPFromPCidV2(pieceCid) + if err != nil { + log.Errorf("parsing piece CID '%s': %s", pieceCidStr, err.Error()) + w.WriteHeader(http.StatusBadRequest) + stats.Record(ctx, remoteblockstore.HttpPieceByCid400ResponseCount.M(1)) + return + } + + pi := commp.PieceInfo() + // Get a reader over the piece - reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pieceCid) + reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) if err != nil { log.Errorf("server error getting content for piece CID %s: %s", pieceCid, err) if errors.Is(err, cachedreader.NoDealErr) { diff --git a/market/retrieval/remoteblockstore/remoteblockstore.go b/market/retrieval/remoteblockstore/remoteblockstore.go index e37bc4dac..9551c7964 100644 --- a/market/retrieval/remoteblockstore/remoteblockstore.go +++ b/market/retrieval/remoteblockstore/remoteblockstore.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" ) @@ -31,7 +32,7 @@ var log = logging.Logger("remote-blockstore") type idxAPI interface { PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]indexstore.PieceInfo, error) - GetOffset(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (uint64, error) + GetOffset(ctx context.Context, pieceCidv2 cid.Cid, hash multihash.Multihash) (uint64, error) } // RemoteBlockstore is a read-only blockstore over all cids across all pieces on a provider. @@ -116,7 +117,12 @@ func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, for _, piece := range pieces { data, err := func() ([]byte, error) { // Get a reader over the piece data - reader, _, err := ro.cpr.GetSharedPieceReader(ctx, piece.PieceCid) + commp, err := commcidv2.CommPFromPCidV2(piece.PieceCidV2) + if err != nil { + return nil, fmt.Errorf("getting commP from piece cid v2 %s: %w", piece.PieceCidV2.String(), err) + } + pi := commp.PieceInfo() + reader, _, err := ro.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) if err != nil { return nil, fmt.Errorf("getting piece reader: %w", err) } @@ -125,19 +131,19 @@ func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, }(reader) // Get the offset of the block within the piece (CAR file) - offset, err := ro.idxApi.GetOffset(ctx, piece.PieceCid, c.Hash()) + offset, err := ro.idxApi.GetOffset(ctx, piece.PieceCidV2, c.Hash()) if err != nil { - return nil, fmt.Errorf("getting offset/size for cid %s in piece %s: %w", c, piece.PieceCid, err) + return nil, fmt.Errorf("getting offset/size for cid %s in piece %s: %w", c, piece.PieceCidV2, err) } // Seek to the section offset readerAt := io.NewSectionReader(reader, int64(offset), int64(piece.BlockSize+MaxCarBlockPrefixSize)) readCid, data, err := util.ReadNode(bufio.NewReader(readerAt)) if err != nil { - return nil, fmt.Errorf("reading data for block %s from reader for piece %s: %w", c, piece.PieceCid, err) + return nil, fmt.Errorf("reading data for block %s from reader for piece %s: %w", c, piece.PieceCidV2, err) } if !bytes.Equal(readCid.Hash(), c.Hash()) { - return nil, fmt.Errorf("read block %s from reader for piece %s, but expected block %s", readCid, piece.PieceCid, c) + return nil, fmt.Errorf("read block %s from reader for piece %s, but expected block %s", readCid, piece.PieceCidV2, c) } return data, nil }() diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index f3d29741b..68f5080ba 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -6,11 +6,19 @@ import ( "errors" "fmt" "io" + "runtime" + "sort" + "strings" + "sync" "time" + "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/go-data-segment/datasegment" + "github.com/filecoin-project/go-data-segment/fr32" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" carv2 "github.com/ipld/go-car/v2" + "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -22,9 +30,10 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" - "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" ) @@ -34,7 +43,7 @@ var log = logging.Logger("indexing") type IndexingTask struct { db *harmonydb.DB indexStore *indexstore.IndexStore - pieceProvider *pieceprovider.SectorReader + cpr *cachedreader.CachedPieceReader sc *ffi.SealCalls cfg *config.CurioConfig insertConcurrency int @@ -42,12 +51,12 @@ type IndexingTask struct { max taskhelp.Limiter } -func NewIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.SectorReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IndexingTask { +func NewIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IndexingTask { return &IndexingTask{ db: db, indexStore: indexStore, - pieceProvider: pieceProvider, + cpr: cpr, sc: sc, cfg: cfg, insertConcurrency: cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency, @@ -141,8 +150,31 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return false, xerrors.Errorf("checking if piece %s is already indexed: %w", task.PieceCid, err) } + var byteData bool + var subPieces []mk20.PieceDataFormat + + if task.Mk20 { + id, err := ulid.Parse(task.UUID) + if err != nil { + return false, xerrors.Errorf("parsing id: %w", err) + } + deal, err := mk20.DealFromDB(ctx, i.db, id) + if err != nil { + return false, xerrors.Errorf("getting mk20 deal from DB: %w", err) + } + if deal.Data.Format.Aggregate != nil { + if deal.Data.Format.Aggregate.Type > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + } + } + + if deal.Data.Format.Raw != nil { + byteData = true + } + } + // Return early if already indexed or should not be indexed - if indexed || !task.ShouldIndex { + if indexed || !task.ShouldIndex || byteData { err = i.recordCompletion(ctx, task, taskID, false) if err != nil { return false, err @@ -157,13 +189,13 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return false, xerrors.Errorf("parsing piece CID: %w", err) } - reader, err := i.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ - ID: abi.SectorID{ - Miner: abi.ActorID(task.SpID), - Number: task.Sector, - }, - ProofType: task.Proof, - }, storiface.PaddedByteIndex(task.Offset).Unpadded(), task.Size.Unpadded(), pieceCid) + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: pieceCid, Size: task.Size}) + if err != nil { + return false, xerrors.Errorf("getting piece commP: %w", err) + } + pc2 := commp.PCidV2() + + reader, _, err := i.cpr.GetSharedPieceReader(ctx, pieceCid, task.Size) if err != nil { return false, xerrors.Errorf("getting piece reader: %w", err) @@ -176,34 +208,182 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do dealCfg := i.cfg.Market.StorageMarketConfig chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize - recs := make(chan indexstore.Record, chanSize) - - //recs := make([]indexstore.Record, 0, chanSize) opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} - blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(reader, 4<<20), opts...) - if err != nil { - return false, fmt.Errorf("getting block reader over piece: %w", err) - } + + recs := make(chan indexstore.Record, chanSize) + var blocks int64 var eg errgroup.Group addFail := make(chan struct{}) var interrupted bool - var blocks int64 - start := time.Now() eg.Go(func() error { defer close(addFail) + return i.indexStore.AddIndex(ctx, pc2, recs) + }) - serr := i.indexStore.AddIndex(ctx, pieceCid, recs) - if serr != nil { - return xerrors.Errorf("adding index to DB: %w", serr) + if task.Mk20 && len(subPieces) > 0 { + blocks, interrupted, err = IndexAggregate(reader, task.Size, subPieces, opts, recs, addFail) + } else { + blocks, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + } + + if err != nil { + // Indexing itself failed, stop early + close(recs) // still safe to close, AddIndex will exit on channel close + // wait for AddIndex goroutine to finish cleanly + _ = eg.Wait() + return false, xerrors.Errorf("indexing failed: %w", err) + } + + // Close the channel + close(recs) + + // Wait till AddIndex is finished + err = eg.Wait() + if err != nil { + return false, xerrors.Errorf("adding index to DB (interrupted %t): %w", interrupted, err) + } + + log.Infof("Indexing deal %s took %0.3f seconds", task.UUID, time.Since(startTime).Seconds()) + + err = i.recordCompletion(ctx, task, taskID, true) + if err != nil { + return false, err + } + + blocksPerSecond := float64(blocks) / time.Since(startTime).Seconds() + log.Infow("Piece indexed", "piece_cid", task.PieceCid, "id", task.UUID, "sp_id", task.SpID, "sector", task.Sector, "blocks", blocks, "blocks_per_second", blocksPerSecond) + + return true, nil +} + +// parseDataSegmentIndex is a local more efficient alternative to the method provided by the datasegment library +func parseDataSegmentIndex(unpaddedReader io.Reader) (datasegment.IndexData, error) { + const ( + unpaddedChunk = 127 + paddedChunk = 128 + ) + + // Read all unpadded data (up to 32 MiB Max as per FRC for 64 GiB sector) + unpaddedData, err := io.ReadAll(unpaddedReader) + if err != nil { + return datasegment.IndexData{}, xerrors.Errorf("reading unpadded data: %w", err) + } + + // Make sure it's aligned to 127 + if len(unpaddedData)%unpaddedChunk != 0 { + return datasegment.IndexData{}, fmt.Errorf("unpadded data length %d is not a multiple of 127", len(unpaddedData)) + } + numChunks := len(unpaddedData) / unpaddedChunk + + // Prepare padded output buffer + paddedData := make([]byte, numChunks*paddedChunk) + + // Parallel pad + var wg sync.WaitGroup + concurrency := runtime.NumCPU() + chunkPerWorker := (numChunks + concurrency - 1) / concurrency + + for w := 0; w < concurrency; w++ { + start := w * chunkPerWorker + end := (w + 1) * chunkPerWorker + if end > numChunks { + end = numChunks } - return nil + wg.Add(1) + go func(start, end int) { + defer wg.Done() + for i := start; i < end; i++ { + in := unpaddedData[i*unpaddedChunk : (i+1)*unpaddedChunk] + out := paddedData[i*paddedChunk : (i+1)*paddedChunk] + fr32.Pad(in, out) + } + }(start, end) + } + wg.Wait() + + // Decode entries + allEntries := make([]datasegment.SegmentDesc, numChunks*2) + for i := 0; i < numChunks; i++ { + p := paddedData[i*paddedChunk : (i+1)*paddedChunk] + + if err := allEntries[i*2+0].UnmarshalBinary(p[:datasegment.EntrySize]); err != nil { + return datasegment.IndexData{}, xerrors.Errorf("unmarshal entry 1 at chunk %d: %w", i, err) + } + if err := allEntries[i*2+1].UnmarshalBinary(p[datasegment.EntrySize:]); err != nil { + return datasegment.IndexData{}, xerrors.Errorf("unmarshal entry 2 at chunk %d: %w", i, err) + } + } + + return datasegment.IndexData{Entries: allEntries}, nil +} + +func validateSegments(segments []datasegment.SegmentDesc) []datasegment.SegmentDesc { + entryCount := len(segments) + + validCh := make(chan datasegment.SegmentDesc, entryCount) + var wg sync.WaitGroup + + workers := runtime.NumCPU() + chunkSize := (entryCount + workers - 1) / workers + + for w := 0; w < workers; w++ { + start := w * chunkSize + end := (w + 1) * chunkSize + if end > entryCount { + end = entryCount + } + if start >= end { + break + } + + wg.Add(1) + go func(start, end int) { + defer wg.Done() + for i := start; i < end; i++ { + entry := segments[i] + if err := entry.Validate(); err == nil { + validCh <- entry + } + log.Debugw("data segment invalid", "segment", entry) + } + }(start, end) + } + + go func() { + wg.Wait() + close(validCh) + }() + + var validEntries []datasegment.SegmentDesc + for entry := range validCh { + validEntries = append(validEntries, entry) + } + sort.Slice(validEntries, func(i, j int) bool { + return validEntries[i].Offset < validEntries[j].Offset }) + return validEntries +} + +func IndexCAR(r io.Reader, buffSize int, opts []carv2.Option, recs chan<- indexstore.Record, addFail <-chan struct{}) (int64, bool, error) { + blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(r, buffSize), opts...) + if err != nil { + return 0, false, fmt.Errorf("getting block reader over piece: %w", err) + } + + var blocks int64 + var interrupted bool + + for { + blockMetadata, err := blockReader.SkipNext() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return blocks, interrupted, fmt.Errorf("generating index for piece: %w", err) + } - blockMetadata, err := blockReader.SkipNext() -loop: - for err == nil { blocks++ select { @@ -214,34 +394,97 @@ loop: }: case <-addFail: interrupted = true - break loop } - blockMetadata, err = blockReader.SkipNext() - } - if err != nil && !errors.Is(err, io.EOF) { - return false, fmt.Errorf("generating index for piece: %w", err) + + if interrupted { + break + } } - // Close the channel - close(recs) + return blocks, interrupted, nil +} - // Wait till AddIndex is finished - err = eg.Wait() +type IndexReader interface { + io.ReaderAt + io.Seeker + io.Reader +} + +func IndexAggregate( + reader IndexReader, + size abi.PaddedPieceSize, + subPieces []mk20.PieceDataFormat, + opts []carv2.Option, + recs chan<- indexstore.Record, + addFail <-chan struct{}, +) (int64, bool, error) { + dsis := datasegment.DataSegmentIndexStartOffset(size) + if _, err := reader.Seek(int64(dsis), io.SeekStart); err != nil { + return 0, false, xerrors.Errorf("seeking to data segment index start offset: %w", err) + } + + idata, err := parseDataSegmentIndex(reader) if err != nil { - return false, xerrors.Errorf("adding index to DB (interrupted %t): %w", interrupted, err) + return 0, false, xerrors.Errorf("parsing data segment index: %w", err) + } + if len(idata.Entries) == 0 { + return 0, false, xerrors.New("no data segment index entries") } - log.Infof("Indexing deal %s took %0.3f seconds", task.UUID, time.Since(startTime).Seconds()) + valid := validateSegments(idata.Entries) + if len(valid) == 0 { + return 0, false, xerrors.New("no valid data segment index entries") + } - err = i.recordCompletion(ctx, task, taskID, true) - if err != nil { - return false, err + var haveSubPieces bool + + if len(subPieces) > 0 { + if len(valid) != len(subPieces) { + return 0, false, xerrors.Errorf("expected %d data segment index entries, got %d", len(subPieces), len(idata.Entries)) + } + haveSubPieces = true } - blocksPerSecond := float64(blocks) / time.Since(start).Seconds() - log.Infow("Piece indexed", "piece_cid", task.PieceCid, "id", task.UUID, "sp_id", task.SpID, "sector", task.Sector, "blocks", blocks, "blocks_per_second", blocksPerSecond) + var totalBlocks int64 + for j, entry := range valid { + bufferSize := 4 << 20 + if entry.Size < uint64(bufferSize) { + bufferSize = int(entry.Size) + } + sectionReader := io.NewSectionReader(reader, int64(entry.Offset), int64(entry.Size)) - return true, nil + b, inter, err := IndexCAR(sectionReader, bufferSize, opts, recs, addFail) + totalBlocks += b + + if err != nil { + if strings.Contains(err.Error(), "invalid car version") { + if haveSubPieces { + if subPieces[j].Car != nil { + return 0, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) + } + if subPieces[j].Raw != nil { + continue + } + if subPieces[j].Aggregate != nil { + b, inter, err = IndexAggregate(sectionReader, abi.PaddedPieceSize(entry.Size), nil, opts, recs, addFail) + if err != nil { + return totalBlocks, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) + } + totalBlocks += b + } + } else { + continue + } + } + return totalBlocks, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) + } + + if inter { + return totalBlocks, true, nil + } + } + + return totalBlocks, false, nil } // recordCompletion add the piece metadata and piece deal to the DB and diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 4c0eae87b..3029e19fc 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -1,17 +1,19 @@ package indexing import ( - "bufio" "bytes" "context" "crypto/rand" + "database/sql" "errors" "fmt" - "io" "net/url" "strings" "time" + "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/market/mk20" + "github.com/google/uuid" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" carv2 "github.com/ipld/go-car/v2" @@ -21,7 +23,9 @@ import ( "github.com/ipni/go-libipni/metadata" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" + "golang.org/x/sync/errgroup" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" @@ -34,7 +38,6 @@ import ( "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" - "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/ipni/chunker" @@ -44,22 +47,22 @@ import ( var ilog = logging.Logger("ipni") type IPNITask struct { - db *harmonydb.DB - indexStore *indexstore.IndexStore - pieceProvider *pieceprovider.SectorReader - sc *ffi.SealCalls - cfg *config.CurioConfig - max taskhelp.Limiter + db *harmonydb.DB + indexStore *indexstore.IndexStore + cpr *cachedreader.CachedPieceReader + sc *ffi.SealCalls + cfg *config.CurioConfig + max taskhelp.Limiter } -func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.SectorReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { +func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { return &IPNITask{ - db: db, - indexStore: indexStore, - pieceProvider: pieceProvider, - sc: sc, - cfg: cfg, - max: max, + db: db, + indexStore: indexStore, + cpr: cpr, + sc: sc, + cfg: cfg, + max: max, } } @@ -68,6 +71,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b var tasks []struct { SPID int64 `db:"sp_id"` + ID sql.NullString `db:"id"` Sector abi.SectorNumber `db:"sector"` Proof abi.RegisteredSealProof `db:"reg_seal_proof"` Offset int64 `db:"sector_offset"` @@ -78,7 +82,8 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b } err = I.db.Select(ctx, &tasks, `SELECT - sp_id, + sp_id, + id, sector, reg_seal_proof, sector_offset, @@ -111,35 +116,93 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, xerrors.Errorf("unmarshaling piece info: %w", err) } - reader, err := I.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ - ID: abi.SectorID{ - Miner: abi.ActorID(task.SPID), - Number: task.Sector, - }, - ProofType: task.Proof, - }, storiface.PaddedByteIndex(task.Offset).Unpadded(), pi.Size.Unpadded(), pi.PieceCID) + reader, _, err := I.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) + if err != nil { return false, xerrors.Errorf("getting piece reader: %w", err) } + defer reader.Close() - opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} - blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(reader, 4<<20), opts...) - if err != nil { - return false, fmt.Errorf("getting block reader over piece: %w", err) + var isMK20 bool + + if task.ID.Valid { + _, err := ulid.Parse(task.ID.String) + if err == nil { + isMK20 = true + } else { + _, err := uuid.Parse(task.ID.String) + if err != nil { + return false, xerrors.Errorf("parsing task id: %w", err) + } + } } + opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} + + recs := make(chan indexstore.Record, 1) + + var eg errgroup.Group + addFail := make(chan struct{}) + var interrupted bool + var subPieces []mk20.PieceDataFormat chk := chunker.NewInitialChunker() - blockMetadata, err := blockReader.SkipNext() - for err == nil { - if err := chk.Accept(blockMetadata.Cid.Hash(), int64(blockMetadata.Offset), blockMetadata.Size+40); err != nil { - return false, xerrors.Errorf("accepting block: %w", err) + eg.Go(func() error { + defer close(addFail) + for rec := range recs { + serr := chk.Accept(rec.Cid.Hash(), int64(rec.Offset), rec.Size) + if serr != nil { + addFail <- struct{}{} + return serr + } } + return nil + }) - blockMetadata, err = blockReader.SkipNext() + if isMK20 { + id, serr := ulid.Parse(task.ID.String) + if serr != nil { + return false, xerrors.Errorf("parsing task id: %w", serr) + } + deal, serr := mk20.DealFromDB(ctx, I.db, id) + if serr != nil { + return false, xerrors.Errorf("getting deal from db: %w", serr) + } + + if deal.Data.Format.Raw != nil { + return false, xerrors.Errorf("raw data not supported") + } + + if deal.Data.Format.Car != nil { + _, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + } + + if deal.Data.Format.Aggregate != nil { + if deal.Data.Format.Aggregate.Type > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + _, interrupted, err = IndexAggregate(reader, pi.Size, subPieces, opts, recs, addFail) + } + } + + } else { + _, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + } + + if err != nil { + // Chunking itself failed, stop early + close(recs) // still safe to close, chk.Accept() will exit on channel close + // wait for chk.Accept() goroutine to finish cleanly + _ = eg.Wait() + return false, xerrors.Errorf("chunking failed: %w", err) } - if !errors.Is(err, io.EOF) { - return false, xerrors.Errorf("reading block: %w", err) + + // Close the channel + close(recs) + + // Wait till is finished + err = eg.Wait() + if err != nil { + return false, xerrors.Errorf("adding index to chunk (interrupted %t): %w", interrupted, err) } // make sure we still own the task before writing to the database @@ -160,7 +223,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b _, err = I.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { var prev string err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev) - if err != nil && err != pgx.ErrNoRows { + if err != nil && !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("querying previous head: %w", err) } @@ -481,7 +544,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun return false, xerrors.Errorf("marshaling piece info: %w", err) } - _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8)`, p.SpID, + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, p.SpID, p.UUID, p.Sector, p.Proof, p.Offset, b.Bytes(), false, pid.String(), id) if err != nil { if harmonydb.IsErrUniqueContraint(err) { diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index a81e8615b..93a71aabb 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -392,7 +392,7 @@ func (p *ProveTask) genSubrootMemtree(ctx context.Context, subrootCid string, su return nil, xerrors.Errorf("subroot size exceeds maximum: %d", subrootSize) } - subrootReader, unssize, err := p.cpr.GetSharedPieceReader(ctx, subrootCidObj) + subrootReader, unssize, err := p.cpr.GetSharedPieceReader(ctx, subrootCidObj, subrootSize) if err != nil { return nil, xerrors.Errorf("failed to get subroot reader: %w", err) } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 0c929caaa..f4869baa2 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -212,7 +212,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 for _, src := range data.SourceHTTP.URLs { var refID int64 - headers, err := json.Marshal(src.HTTPHeaders) + headers, err := json.Marshal(src.Headers) if err != nil { return xerrors.Errorf("marshaling headers: %w", err) } @@ -377,7 +377,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 var refIds []int64 urls := toDownload[downloadkey{PieceCID: k.PieceCID, Size: k.Size}] for _, src := range urls { - headers, err := json.Marshal(src.HTTPHeaders) + headers, err := json.Marshal(src.Headers) if err != nil { return xerrors.Errorf("marshal headers: %w", err) } diff --git a/web/api/webrpc/deals.go b/web/api/webrpc/deals.go index 2648eae75..6b41c436a 100644 --- a/web/api/webrpc/deals.go +++ b/web/api/webrpc/deals.go @@ -4,8 +4,11 @@ import ( "context" "time" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/curio/market/storageingest" @@ -22,6 +25,7 @@ type OpenDealInfo struct { PieceSizeStr string `db:"-"` CreatedAtStr string `db:"-"` + PieceCidV2 string `db:"-"` Miner string } @@ -41,6 +45,15 @@ func (a *WebRPC) DealsPending(ctx context.Context) ([]OpenDealInfo, error) { return nil, err } deals[i].Miner = maddr.String() + pcid, err := cid.Parse(deals[i].PieceCID) + if err != nil { + return nil, xerrors.Errorf("failed to parse piece cid: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: pcid, Size: abi.PaddedPieceSize(deals[i].PieceSize)}) + if err != nil { + return nil, xerrors.Errorf("failed to get commp: %w", err) + } + deals[i].PieceCidV2 = commp.PCidV2().String() } return deals, nil diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index 9ae0e1f3c..3281cdaec 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -36,7 +37,8 @@ type IpniAd struct { EntryCount int64 `json:"entry_count"` CIDCount int64 `json:"cid_count"` - AdCids []string `db:"-" json:"ad_cids"` + AdCids []string `db:"-" json:"ad_cids"` + PieceCidV2 string `db:"-" json:"piece_cid_v2"` } func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { @@ -96,9 +98,15 @@ func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { return nil, xerrors.Errorf("failed to unmarshal piece info: %w", err) } + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return nil, xerrors.Errorf("failed to get commp: %w", err) + } + details.PieceCid = pi.PieceCID.String() size := int64(pi.Size) details.PieceSize = size + details.PieceCidV2 = commp.PCidV2().String() maddr, err := address.NewIDAddress(uint64(details.SpID)) if err != nil { diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index d496c0ef3..3f173e536 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/market/mk20" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/samber/lo" @@ -93,6 +95,7 @@ type MK12Pipeline struct { Started bool `db:"started" json:"started"` PieceCid string `db:"piece_cid" json:"piece_cid"` PieceSize int64 `db:"piece_size" json:"piece_size"` + PieceCidV2 string `db:"-" json:"piece_cid_v2"` RawSize *int64 `db:"raw_size" json:"raw_size"` Offline bool `db:"offline" json:"offline"` URL *string `db:"url" json:"url"` @@ -113,7 +116,7 @@ type MK12Pipeline struct { Miner string `json:"miner"` } -func (a *WebRPC) GetDealPipelines(ctx context.Context, limit int, offset int) ([]*MK12Pipeline, error) { +func (a *WebRPC) GetMK12DealPipelines(ctx context.Context, limit int, offset int) ([]*MK12Pipeline, error) { if limit <= 0 { limit = 25 } @@ -163,6 +166,18 @@ func (a *WebRPC) GetDealPipelines(ctx context.Context, limit int, offset int) ([ return nil, xerrors.Errorf("failed to parse the miner ID: %w", err) } s.Miner = addr.String() + pcid, err := cid.Parse(s.PieceCid) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(s.PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + s.PieceCidV2 = commp.PCidV2().String() } return pipelines, nil @@ -382,14 +397,15 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal } type StorageDealList struct { - ID string `db:"uuid" json:"id"` - MinerID int64 `db:"sp_id" json:"sp_id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - PieceCid string `db:"piece_cid" json:"piece_cid"` - PieceSize int64 `db:"piece_size" json:"piece_size"` - Processed bool `db:"processed" json:"processed"` - Error sql.NullString `db:"error" json:"error"` - Miner string `json:"miner"` + ID string `db:"uuid" json:"id"` + MinerID int64 `db:"sp_id" json:"sp_id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + PieceCidV1 string `db:"piece_cid" json:"piece_cid"` + PieceSize int64 `db:"piece_size" json:"piece_size"` + PieceCidV2 string `json:"piece_cid_v2"` + Processed bool `db:"processed" json:"processed"` + Error sql.NullString `db:"error" json:"error"` + Miner string `json:"miner"` } func (a *WebRPC) MK12StorageDealList(ctx context.Context, limit int, offset int) ([]*StorageDealList, error) { @@ -417,6 +433,18 @@ func (a *WebRPC) MK12StorageDealList(ctx context.Context, limit int, offset int) return nil, err } mk12Summaries[i].Miner = addr.String() + pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() } return mk12Summaries, nil @@ -446,6 +474,18 @@ func (a *WebRPC) LegacyStorageDealList(ctx context.Context, limit int, offset in return nil, err } mk12Summaries[i].Miner = addr.String() + pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() } return mk12Summaries, nil } @@ -599,9 +639,16 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er return nil, err } + commp, err := commcidv2.CommPFromPCidV2(piece) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece CID: %w", err) + } + + pi := commp.PieceInfo() + ret := &PieceInfo{} - err = a.deps.DB.QueryRow(ctx, `SELECT created_at, indexed, indexed_at FROM market_piece_metadata WHERE piece_cid = $1`, piece.String()).Scan(&ret.CreatedAt, &ret.Indexed, &ret.IndexedAT) + err = a.deps.DB.QueryRow(ctx, `SELECT created_at, indexed, indexed_at FROM market_piece_metadata WHERE piece_cid = $1 AND piece_size = $2`, pi.PieceCID.String(), pi.Size).Scan(&ret.CreatedAt, &ret.Indexed, &ret.IndexedAT) if err != nil && err != pgx.ErrNoRows { return nil, xerrors.Errorf("failed to get piece metadata: %w", err) } @@ -619,7 +666,7 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er piece_length, raw_size FROM market_piece_deal - WHERE piece_cid = $1`, piece.String()) + WHERE piece_cid = $1 AND piece_length = $2`, pi.PieceCID.String(), pi.Size) if err != nil { return nil, xerrors.Errorf("failed to get piece deals: %w", err) } @@ -635,11 +682,6 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er ret.Deals = pieceDeals ret.PieceCid = piece.String() - pi := abi.PieceInfo{ - PieceCID: piece, - Size: abi.PaddedPieceSize(ret.Size), - } - b := new(bytes.Buffer) err = pi.MarshalCBOR(b) @@ -679,13 +721,25 @@ type ParkedPieceRef struct { // PieceParkStates retrieves the park states for a given piece CID func (a *WebRPC) PieceParkStates(ctx context.Context, pieceCID string) (*ParkedPieceState, error) { + pcid, err := cid.Parse(pieceCID) + if err != nil { + return nil, err + } + + commp, err := commcidv2.CommPFromPCidV2(pcid) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece CID: %w", err) + } + + pi := commp.PieceInfo() + var pps ParkedPieceState // Query the parked_pieces table - err := a.deps.DB.QueryRow(ctx, ` + err = a.deps.DB.QueryRow(ctx, ` SELECT id, created_at, piece_cid, piece_padded_size, piece_raw_size, complete, task_id, cleanup_task_id - FROM parked_pieces WHERE piece_cid = $1 - `, pieceCID).Scan( + FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND piece_raw_size = $3 + `, pi.PieceCID.String(), pi.Size, commp.PayloadSize()).Scan( &pps.ID, &pps.CreatedAt, &pps.PieceCID, &pps.PiecePaddedSize, &pps.PieceRawSize, &pps.Complete, &pps.TaskID, &pps.CleanupTaskID, ) @@ -798,16 +852,72 @@ type MK12DealPipeline struct { CreatedAt time.Time `db:"created_at" json:"created_at"` } -// MK12DealDetailEntry combines a deal and its pipeline -type MK12DealDetailEntry struct { - Deal *MK12Deal `json:"deal"` - Pipeline *MK12DealPipeline `json:"pipeline,omitempty"` +// MK20DealPipeline represents a record from market_mk12_deal_pipeline table +type MK20DealPipeline struct { + ID string `db:"id" json:"id"` + SpId int64 `db:"sp_id" json:"sp_id"` + Contract string `db:"contract" json:"contract"` + Client string `db:"client" json:"client"` + PieceCid string `db:"piece_cid" json:"piece_cid"` + PieceSize int64 `db:"piece_size" json:"piece_size"` + RawSize sql.NullInt64 `db:"raw_size" json:"raw_size"` + Offline bool `db:"offline" json:"offline"` + URL sql.NullString `db:"url" json:"url"` + Indexing bool `db:"indexing" json:"indexing"` + Announce bool `db:"announce" json:"announce"` + AllocationID sql.NullInt64 `db:"allocation_id" json:"allocation_id"` + Duration int64 `db:"duration" json:"duration"` + PieceAggregation int `db:"piece_aggregation" json:"piece_aggregation"` + + Started bool `db:"started" json:"started"` + Downloaded bool `db:"downloaded" json:"downloaded"` + + CommpTaskId sql.NullInt64 `db:"commp_task_id" json:"commp_task_id"` + AfterCommp bool `db:"after_commp" json:"after_commp"` + + DealAggregation int `db:"deal_aggregation" json:"deal_aggregation"` + AggregationIndex int64 `db:"aggr_index" json:"aggr_index"` + AggregationTaskID sql.NullInt64 `db:"agg_task_id" json:"agg_task_id"` + Aggregated bool `db:"aggregated" json:"aggregated"` + + Sector sql.NullInt64 `db:"sector" json:"sector"` + RegSealProof sql.NullInt64 `db:"reg_seal_proof" json:"reg_seal_proof"` + SectorOffset sql.NullInt64 `db:"sector_offset" json:"sector_offset"` + Sealed sql.NullBool `db:"sealed" json:"sealed"` + + IndexingCreatedAt sql.NullTime `db:"indexing_created_at" json:"indexing_created_at"` + IndexingTaskId sql.NullInt64 `db:"indexing_task_id" json:"indexing_task_id"` + Indexed sql.NullBool `db:"indexed" json:"indexed"` + + Complete bool `db:"complete" json:"complete"` + CreatedAt time.Time `db:"created_at" json:"created_at"` } -func (a *WebRPC) MK12DealDetail(ctx context.Context, pieceCid string) ([]MK12DealDetailEntry, error) { +// PieceDealDetailEntry combines a deal and its pipeline +type PieceDealDetailEntry struct { + MK12Deal *MK12Deal `json:"mk12_deal"` + MK12Pipeline *MK12DealPipeline `json:"mk12_pipeline,omitempty"` + MK20Deal *mk20.Deal `json:"mk20_deal,omitempty"` + MK20DealPipeline *MK20DealPipeline `json:"mk20_pipeline,omitempty"` +} + +func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceDealDetailEntry, error) { + pcid, err := cid.Parse(pieceCid) + if err != nil { + return nil, err + } + + commp, err := commcidv2.CommPFromPCidV2(pcid) + if err != nil { + return nil, err + } + + pieceCid = commp.PieceInfo().PieceCID.String() + size := commp.PieceInfo().Size + var mk12Deals []*MK12Deal - err := a.deps.DB.Select(ctx, &mk12Deals, ` + err = a.deps.DB.Select(ctx, &mk12Deals, ` SELECT uuid, sp_id, @@ -832,7 +942,7 @@ func (a *WebRPC) MK12DealDetail(ctx context.Context, pieceCid string) ([]MK12Dea error, FALSE AS is_ddo FROM market_mk12_deals - WHERE piece_cid = $1 + WHERE piece_cid = $1 AND piece_size = $2 UNION ALL @@ -860,7 +970,7 @@ func (a *WebRPC) MK12DealDetail(ctx context.Context, pieceCid string) ([]MK12Dea NULL AS error, -- NULL handled by Go (sql.NullString) TRUE AS is_ddo FROM market_direct_deals - WHERE piece_cid = $1`, pieceCid) + WHERE piece_cid = $1 AND piece_size = $2`, pieceCid, size) if err != nil { return nil, err } @@ -909,7 +1019,7 @@ func (a *WebRPC) MK12DealDetail(ctx context.Context, pieceCid string) ([]MK12Dea WHERE uuid = ANY($1) `, uuids) if err != nil { - return nil, err + return nil, xerrors.Errorf("failed to query mk12 pipelines: %w", err) } } @@ -919,15 +1029,90 @@ func (a *WebRPC) MK12DealDetail(ctx context.Context, pieceCid string) ([]MK12Dea pipelineMap[pipeline.UUID] = pipeline } - var entries []MK12DealDetailEntry + var entries []PieceDealDetailEntry for _, deal := range mk12Deals { - entry := MK12DealDetailEntry{ - Deal: deal, + entry := PieceDealDetailEntry{ + MK12Deal: deal, } if pipeline, exists := pipelineMap[deal.UUID]; exists { - entry.Pipeline = &pipeline + entry.MK12Pipeline = &pipeline } else { - entry.Pipeline = nil // Pipeline may not exist for processed and active deals + entry.MK12Pipeline = nil // Pipeline may not exist for processed and active deals + } + entries = append(entries, entry) + } + + var mk20Deals []*mk20.DBDeal + err = a.deps.DB.Select(ctx, &mk20Deals, `SELECT * FROM market_mk20_deals WHERE piece_cid = $1 AND piece_size = $2`, pieceCid) + if err != nil { + return nil, xerrors.Errorf("failed to query mk20 deals: %w", err) + } + + ids := make([]string, len(mk20Deals)) + mk20deals := make([]*mk20.Deal, len(mk20Deals)) + + for i, dbdeal := range mk20Deals { + deal, err := dbdeal.ToDeal() + if err != nil { + return nil, err + } + ids[i] = deal.Identifier.String() + mk20deals[i] = deal + } + + var mk20Pipelines []*MK12DealPipeline + err = a.deps.DB.Select(ctx, &mk20Pipelines, ` + SELECT + created_at, + id, + sp_id, + contract, + client, + piece_cid, + piece_size, + raw_size, + offline, + url, + indexing, + announce, + allocation_id, + piece_aggregation, + started, + downloaded, + commp_task_id, + after_commp, + deal_aggregation, + aggr_index, + agg_task_id, + aggregated, + sector, + reg_seal_proof, + sector_offset, + sealed, + indexing_created_at, + indexing_task_id, + indexed, + complete + FROM market_mk20_pipeline + WHERE id = ANY($1)`) + if err != nil { + return nil, xerrors.Errorf("failed to query mk20 pipelines: %w", err) + } + + mk20pipelineMap := make(map[string]MK20DealPipeline) + for _, pipeline := range mk20pipelineMap { + pipeline := pipeline + mk20pipelineMap[pipeline.ID] = pipeline + } + + for _, deal := range mk20deals { + entry := PieceDealDetailEntry{ + MK20Deal: deal, + } + if pipeline, exists := mk20pipelineMap[deal.Identifier.String()]; exists { + entry.MK20DealPipeline = &pipeline + } else { + entry.MK20DealPipeline = nil // Pipeline may not exist for processed and active deals } entries = append(entries, entry) } @@ -1045,7 +1230,7 @@ type PipelineFailedStats struct { IndexFailed int64 } -func (a *WebRPC) PipelineFailedTasksMarket(ctx context.Context) (*PipelineFailedStats, error) { +func (a *WebRPC) MK12PipelineFailedTasks(ctx context.Context) (*PipelineFailedStats, error) { // We'll create a similar query, but this time we coalesce the task IDs from harmony_task. // If the join fails (no matching harmony_task), all joined fields for that task will be NULL. // We detect failure by checking that xxx_task_id IS NOT NULL, after_xxx = false, and that no task record was found in harmony_task. @@ -1475,6 +1660,18 @@ func (a *WebRPC) MK12DDOStorageDealList(ctx context.Context, limit int, offset i return nil, err } mk12Summaries[i].Miner = addr.String() + pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() } return mk12Summaries, nil diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go new file mode 100644 index 000000000..1f73c6dc4 --- /dev/null +++ b/web/api/webrpc/market_20.go @@ -0,0 +1,89 @@ +package webrpc + +import ( + "context" + "database/sql" + "fmt" + + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "golang.org/x/xerrors" +) + +type MK20StorageDeal struct { + Deal *mk20.Deal `json:"deal"` + Error sql.NullString `json:"error"` +} + +func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, idStr string) (*MK20StorageDeal, error) { + id, err := ulid.Parse(idStr) + if err != nil { + return nil, xerrors.Errorf("parsing deal ID: %w", err) + } + + var dbDeal []mk20.DBDeal + err = a.deps.DB.Select(ctx, &dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) + if err != nil { + return nil, xerrors.Errorf("getting deal from DB: %w", err) + } + if len(dbDeal) != 1 { + return nil, xerrors.Errorf("expected 1 deal, got %d", len(dbDeal)) + } + deal, err := dbDeal[0].ToDeal() + if err != nil { + return nil, xerrors.Errorf("converting DB deal to struct: %w", err) + } + + return &MK20StorageDeal{Deal: deal, Error: dbDeal[0].Error}, nil +} + +func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) ([]*StorageDealList, error) { + var mk20Summaries []*StorageDealList + + err := a.deps.DB.Select(ctx, &mk20Summaries, `SELECT + d.id as uuid, + d.piece_cid, + d.size AS piece_size, + d.created_at, + d.sp_id, + d.error, + CASE + WHEN w.id IS NOT NULL THEN FALSE + WHEN p.id IS NOT NULL THEN p.complete + ELSE TRUE + END AS processed + FROM market_mk20_deal d + LEFT JOIN market_mk20_pipeline_waiting w ON d.id = w.id + LEFT JOIN market_mk20_pipeline p ON d.id = p.id + WHERE d.ddo_v1 IS NOT NULL AND d.ddo_v1 != 'null' + ORDER BY d.created_at DESC + LIMIT $1 OFFSET $2;`, limit, offset) + if err != nil { + return nil, fmt.Errorf("failed to fetch deal list: %w", err) + } + + for i := range mk20Summaries { + addr, err := address.NewIDAddress(uint64(mk20Summaries[i].MinerID)) + if err != nil { + return nil, err + } + mk20Summaries[i].Miner = addr.String() + pcid, err := cid.Parse(mk20Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(mk20Summaries[i].PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + mk20Summaries[i].PieceCidV2 = commp.PCidV2().String() + } + return mk20Summaries, nil +} diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index 807c4d06d..0e851bbae 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -7,6 +7,8 @@ import ( "time" "github.com/docker/go-units" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/ipfs/go-cid" "github.com/samber/lo" "github.com/snadrus/must" "golang.org/x/xerrors" @@ -115,6 +117,7 @@ type SectorPieceMeta struct { PieceIndex int64 `db:"piece_index"` PieceCid string `db:"piece_cid"` PieceSize int64 `db:"piece_size"` + PieceCidV2 string `db:"-"` DealID *string `db:"deal_id"` DataUrl *string `db:"data_url"` @@ -513,6 +516,23 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto pieces[i].StrPieceSize = types.SizeStr(types.NewInt(uint64(pieces[i].PieceSize))) pieces[i].StrDataRawSize = types.SizeStr(types.NewInt(uint64(derefOrZero(pieces[i].DataRawSize)))) + pcid, err := cid.Parse(pieces[i].PieceCid) + if err != nil { + return nil, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pi := abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(uint64(pieces[i].PieceSize)), + } + + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return nil, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pieces[i].PieceCidV2 = commp.PCidV2().String() + id, isPiecePark := strings.CutPrefix(derefOrZero(pieces[i].DataUrl), "pieceref:") if !isPiecePark { continue diff --git a/web/static/pages/ipni/ipni_search.mjs b/web/static/pages/ipni/ipni_search.mjs index 7f26412dd..a645fe07e 100644 --- a/web/static/pages/ipni/ipni_search.mjs +++ b/web/static/pages/ipni/ipni_search.mjs @@ -167,7 +167,7 @@ class IpniSearch extends LitElement { Piece CID - ${this.adData.piece_cid} + ${this.adData.piece_cid} Piece Size diff --git a/web/static/pages/market/index.html b/web/static/pages/market/index.html index 659815ace..98f8e81de 100644 --- a/web/static/pages/market/index.html +++ b/web/static/pages/market/index.html @@ -4,8 +4,6 @@ Storage Market - - @@ -29,13 +27,6 @@

Storage Market

-
-
-
- -
-
-
@@ -43,13 +34,6 @@

Storage Market

-
-
-
- -
-
-
diff --git a/web/static/pages/market/pending-deals.mjs b/web/static/pages/market/pending-deals.mjs index f22f6e558..cdac22c5e 100644 --- a/web/static/pages/market/pending-deals.mjs +++ b/web/static/pages/market/pending-deals.mjs @@ -58,7 +58,7 @@ class PendingDeals extends LitElement { ${entry.Miner} ${entry.SectorNumber} - ${entry.PieceCID} + ${entry.PieceCID} ${entry.PieceSizeStr} ${entry.CreatedAtStr} diff --git a/web/static/pages/mk12-deal/deal.mjs b/web/static/pages/mk12-deal/deal.mjs index 5d8d3fb26..1c06cce8d 100644 --- a/web/static/pages/mk12-deal/deal.mjs +++ b/web/static/pages/mk12-deal/deal.mjs @@ -40,7 +40,7 @@ class DealDetails extends LitElement { {property: 'Client Peer ID', value: html``}, {property: 'Chain Deal ID', value: entry.chain_deal_id}, {property: 'Publish CID', value: entry.publish_cid}, - {property: 'Piece CID', value: html`${entry.piece_cid}`}, + {property: 'Piece CID', value: html`${entry.piece_cid}`}, {property: 'Piece Size', value: entry.piece_size}, {property: 'Fast Retrieval', value: entry.fast_retrieval}, {property: 'Announce To IPNI', value: entry.announce_to_ipni}, diff --git a/web/static/pages/mk12-deal/index.html b/web/static/pages/mk12-deal/index.html index 5f63a80b5..6d03731eb 100644 --- a/web/static/pages/mk12-deal/index.html +++ b/web/static/pages/mk12-deal/index.html @@ -17,7 +17,7 @@
-

Deal Info

+

MK12 Deal Info

diff --git a/web/static/pages/market/deal-pipelines.mjs b/web/static/pages/mk12-deals/deal-pipelines.mjs similarity index 98% rename from web/static/pages/market/deal-pipelines.mjs rename to web/static/pages/mk12-deals/deal-pipelines.mjs index c5b09f63a..df1a135c2 100644 --- a/web/static/pages/market/deal-pipelines.mjs +++ b/web/static/pages/mk12-deals/deal-pipelines.mjs @@ -40,11 +40,11 @@ class DealPipelines extends LitElement { async loadData() { try { const params = [this.limit, this.offset]; - const deals = await RPCCall('GetDealPipelines', params); + const deals = await RPCCall('GetMK12DealPipelines', params); this.deals = deals; // Load failed tasks data - const failed = await RPCCall('PipelineFailedTasksMarket', []); + const failed = await RPCCall('MK12PipelineFailedTasks', []); this.failedTasks = failed || {}; this.requestUpdate(); @@ -224,7 +224,7 @@ class DealPipelines extends LitElement { ${deal.miner} - ${this.formatPieceCid(deal.piece_cid)} + ${this.formatPieceCid(deal.piece_cid)} ${this.formatBytes(deal.piece_size)} ${this.getDealStatus(deal)} diff --git a/web/static/pages/mk12-deals/index.html b/web/static/pages/mk12-deals/index.html index 5f65ce17b..dd673b3e2 100644 --- a/web/static/pages/mk12-deals/index.html +++ b/web/static/pages/mk12-deals/index.html @@ -5,6 +5,8 @@ + + @@ -22,6 +24,20 @@

Storage Deals

+
+
+
+ +
+
+
+
+
+
+ +
+
+
diff --git a/web/static/pages/market/market-asks.mjs b/web/static/pages/mk12-deals/market-asks.mjs similarity index 83% rename from web/static/pages/market/market-asks.mjs rename to web/static/pages/mk12-deals/market-asks.mjs index 670b995df..70d926260 100644 --- a/web/static/pages/market/market-asks.mjs +++ b/web/static/pages/mk12-deals/market-asks.mjs @@ -161,56 +161,56 @@ class MarketAsks extends LitElement { render() { return html` - - - -
-

Storage Asks

- - - - - - - - - - - - - - - - ${this.actorList.map((spID) => { - const ask = this.spAsks.get(spID); - return html` - - - - - - - - - - - - `; - })} - -
SP IDPrice (FIL/TiB/Month)Price (attoFIL/GiB/Epoch)Verified Price (FIL/TiB/Month)Verified Price (attoFIL/GiB/Epoch)Min SizeMax SizeSequenceActions
${ask ? ask.Miner : ''}${ask ? this.attoFilToFilPerTiBPerMonth(ask.Price) : '-'}${ask ? ask.Price : '-'}${ask ? this.attoFilToFilPerTiBPerMonth(ask.VerifiedPrice) : '-'}${ask ? ask.VerifiedPrice : '-'}${ask ? this.formatBytes(ask.MinSize) : '-'}${ask ? this.formatBytes(ask.MaxSize) : '-'}${ask ? ask.Sequence : '-'} - -
- ${this.updatingSpID !== null ? this.renderUpdateForm() : ''} -
- `; + + + +
+

Storage Asks

+ + + + + + + + + + + + + + + + ${this.actorList.map((spID) => { + const ask = this.spAsks.get(spID); + return html` + + + + + + + + + + + + `; + })} + +
SP IDPrice (FIL/TiB/Month)Price (attoFIL/GiB/Epoch)Verified Price (FIL/TiB/Month)Verified Price (attoFIL/GiB/Epoch)Min SizeMax SizeSequenceActions
${ask ? ask.Miner : ''}${ask ? this.attoFilToFilPerTiBPerMonth(ask.Price) : '-'}${ask ? ask.Price : '-'}${ask ? this.attoFilToFilPerTiBPerMonth(ask.VerifiedPrice) : '-'}${ask ? ask.VerifiedPrice : '-'}${ask ? this.formatBytes(ask.MinSize) : '-'}${ask ? this.formatBytes(ask.MaxSize) : '-'}${ask ? ask.Sequence : '-'} + +
+ ${this.updatingSpID !== null ? this.renderUpdateForm() : ''} +
+ `; } renderUpdateForm() { diff --git a/web/static/pages/mk12-deals/mk12-deals.mjs b/web/static/pages/mk12-deals/mk12-deals.mjs index 74b502525..c9c5a1c1b 100644 --- a/web/static/pages/mk12-deals/mk12-deals.mjs +++ b/web/static/pages/mk12-deals/mk12-deals.mjs @@ -92,7 +92,7 @@ class MK12DealList extends LitElement { ${formatDate(deal.created_at)} ${deal.id} ${deal.miner} - ${deal.piece_cid} + ${deal.piece_cid} ${this.formatBytes(deal.piece_size)} diff --git a/web/static/pages/mk12-deals/mk12ddo-list.mjs b/web/static/pages/mk12-deals/mk12ddo-list.mjs index 1ccf283a3..3fb34796f 100644 --- a/web/static/pages/mk12-deals/mk12ddo-list.mjs +++ b/web/static/pages/mk12-deals/mk12ddo-list.mjs @@ -82,7 +82,6 @@ class MK12DDODealList extends LitElement { Piece Size Processed Error - @@ -92,7 +91,7 @@ class MK12DDODealList extends LitElement { ${formatDate(deal.created_at)} ${deal.id} ${deal.miner} - ${deal.piece_cid} + ${deal.piece_cid} ${this.formatBytes(deal.piece_size)} diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs new file mode 100644 index 000000000..f08232b0a --- /dev/null +++ b/web/static/pages/mk20-deal/deal.mjs @@ -0,0 +1,256 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; +import { formatDate } from '/lib/dateutil.mjs'; +import '/ux/epoch.mjs'; +import '/lib/cu-wallet.mjs'; +import '/ux/yesno.mjs'; + +class DealDetails extends LitElement { + constructor() { + super(); + this.loadData(); + } + + async loadData() { + try { + const params = new URLSearchParams(window.location.search); + this.data = await RPCCall('MK20DDOStorageDeal', [params.get('id')]); + setTimeout(() => this.loadData(), 10000); + this.requestUpdate(); + } catch (error) { + alert('Failed to load deal details: ' + error); + console.error('Failed to load deal details:', error); + } + } + + render() { + if (!this.data) return html`

No data.

`; + + const { Identifier, Data, Products } = this.data.deal; + + return html` +
+
Deal
+ + + + + +
Identifier${Identifier}
Error
PieceCID${Data?.piece_cid['/']}
Size${Data?.size}
+ + ${this.renderPieceFormat(Data?.format)} + ${Data?.source_http ? this.renderSourceHTTP(Data.source_http) : ''} + ${Data?.source_aggregate ? this.renderSourceAggregate(Data.source_aggregate) : ''} + ${Data?.source_offline ? this.renderSourceOffline(Data.source_offline) : ''} + ${Data?.source_httpput ? this.renderSourceHttpPut(Data.source_httpput) : ''} + + ${Products?.ddo_v1 ? this.renderDDOV1(Products.ddo_v1) : ''} +
+ `; + } + + renderPieceFormat(format) { + if (!format) return ''; + return html` +
Piece Format
+ + ${format.car ? html`` : ''} + ${format.aggregate + ? html` + + + ` + : ''} + ${format.raw ? html`` : ''} +
CarYes
Aggregate Type${format.aggregate.type}
${this.renderAggregateSubs(format.aggregate.sub)}
RawYes
+ `; + } + + renderAggregateSubs(subs) { + if (!subs?.length) return ''; + return html` +
Aggregate Sub Formats
+ + + + ${subs.map((s, i) => html` + + + + + + + `)} + +
#CarRawAggregate
${i + 1}${s.car ? 'Yes' : ''}${s.raw ? 'Yes' : ''}${s.aggregate ? 'Yes' : ''}
+ `; + } + + renderSourceHTTP(src) { + return html` +
Source HTTP
+ + + + + +
Raw Size${src.rawsize}
+ URLs + + + + ${src.urls.map(u => html` + + + + + + `)} + +
URLPriorityFallback
${u.url}${u.priority}${u.fallback ? 'Yes' : 'No'}
+
+ `; + } + + renderSourceAggregate(src) { + return html` +
Source Aggregate
+ ${src.pieces.map((piece, i) => html` +
+ Piece ${i + 1} + + + +
PieceCID${piece.piece_cid['/']}
Size${piece.size}
+
+ `)} + `; + } + + renderSourceOffline(src) { + return html` +
Source Offline
+ + +
Raw Size${src.raw_size}
+ `; + } + + renderSourceHttpPut(src) { + return html` +
Source HTTP PUT
+ + +
Raw Size${src.raw_size}
+ `; + } + + renderDDOV1(ddo) { + return html` +
DDO v1
+ + + + + + ${ddo.allocation_id ? html`` : ''} + + + + + +
Provider${ddo.provider}
Client${ddo.client}
Piece Manager${ddo.piece_manager}
Duration${ddo.duration}
Allocation ID${ddo.allocation_id}
Contract${ddo.contract_address}
Verify Method${ddo.contract_verify_method}
Notify Address${ddo.notification_address}
Indexing${ddo.indexing ? 'Yes' : 'No'}
Announce to IPNI${ddo.announce_to_ipni ? 'Yes' : 'No'}
+ `; + } +} +customElements.define('deal-details', DealDetails); + +// import { LitElement, html, css } from 'lit'; +// import { customElement, property } from 'lit/decorators.js'; +// +// @customElement('deal-view') +// export class DealView extends LitElement { +// @property({ type: Object }) deal; +// +// static styles = css` +// table { +// border-collapse: collapse; +// width: 100%; +// margin-bottom: 1rem; +// } +// th, td { +// border: 1px solid #ddd; +// padding: 0.5rem; +// vertical-align: top; +// } +// th { +// background-color: #f8f9fa; +// text-align: left; +// } +// .nested-table { +// margin-left: 1rem; +// width: auto; +// } +// `; +// +// renderNested(title, obj) { +// if (!obj) return html``; +// return html` +// +// ${title} +// +// ${Object.entries(obj).map(([key, value]) => html` +// +// ${key} +// +// ${typeof value === 'object' && value !== null +// ? html`${this.renderRows(value)}
` +// : String(value)} +// +// +// `)} +// `; +// } +// +// renderRows(data) { +// return Object.entries(data).map(([key, value]) => { +// if (typeof value === 'object' && value !== null && !Array.isArray(value)) { +// return html`${this.renderNested(key, value)}`; +// } else { +// return html` +// +// ${key} +// ${Array.isArray(value) ? html`
${JSON.stringify(value, null, 2)}
` : String(value)} +// +// `; +// } +// }); +// } +// +// render() { +// if (!this.deal) return html`

No deal provided.

`; +// return html` +// +// +// +// +// +// +// ${this.deal.data ? html` +// +// +// +// ${this.renderNested('Data', this.deal.data)} +// ` : null} +// ${this.deal.products?.ddo_v1 ? html` +// +// +// +// ${this.renderNested('DDOV1', this.deal.products.ddo_v1)} +// ` : null} +// +//
Deal
Identifier${this.deal.identifier}
Data
DDOV1
+// `; +// } +// } + diff --git a/web/static/pages/mk20-deal/index.html b/web/static/pages/mk20-deal/index.html new file mode 100644 index 000000000..f234b04ee --- /dev/null +++ b/web/static/pages/mk20-deal/index.html @@ -0,0 +1,36 @@ + + + + + Deals + + + + + + + + +
+
+
+
+

MK20 Deal Info

+
+
+
+
+
+
+ +
+
+
+
+ + + + + \ No newline at end of file diff --git a/web/static/pages/mk20/ddo.mjs b/web/static/pages/mk20/ddo.mjs new file mode 100644 index 000000000..bec04db79 --- /dev/null +++ b/web/static/pages/mk20/ddo.mjs @@ -0,0 +1,177 @@ +import {css, html, LitElement} from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; +import { formatDate } from '/lib/dateutil.mjs'; +import '/ux/yesno.mjs'; + +class MK20DDODealList extends LitElement { + static properties = { + deals: { type: Array }, + limit: { type: Number }, + offset: { type: Number }, + totalCount: { type: Number }, + }; + + constructor() { + super(); + this.deals = []; + this.limit = 25; + this.offset = 0; + this.totalCount = 0; + this.loadData(); + } + + async loadData() { + try { + const params = [this.limit, this.offset]; + this.deals = await RPCCall('MK20DDOStorageDeals', params); + this.requestUpdate(); + } catch (error) { + console.error('Failed to load ddo deals:', error); + } + } + + nextPage() { + this.offset += this.limit; + this.loadData(); + } + + prevPage() { + if (this.offset >= this.limit) { + this.offset -= this.limit; + } else { + this.offset = 0; + } + this.loadData(); + } + + render() { + // Check if there's an error or if the deals array is empty + if (!this.deals || this.deals.length === 0) { + return html``; // Return an empty template if there's no data to render + } + + return html` + + + +
+

DDO Deal List + +

+ + + + + + + + + + + + + + + ${this.deals.map( + (deal) => html` + + + + + + + + + + ` + )} + +
Created AtIDProviderPiece CIDPiece SizeProcessedError
${formatDate(deal.created_at)}${deal.id}${deal.miner}${deal.piece_cid}${this.formatBytes(deal.piece_size)}
+
+ + Page ${(this.offset / this.limit) + 1} + +
+
+ `; + } + + formatBytes(bytes) { + const units = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + let size = bytes; + while (size >= 1024 && i < units.length - 1) { + size /= 1024; + i++; + } + if (i === 0) { + return `${size} ${units[i]}`; + } else { + return `${size.toFixed(2)} ${units[i]}`; + } + } + + static styles = css` + .pagination-controls { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + + .info-btn { + position: relative; + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + font-size: 1em; + margin-left: 8px; + } + + .tooltip-text { + display: none; + position: absolute; + top: 50%; + left: 120%; /* Position the tooltip to the right of the button */ + transform: translateY(-50%); /* Center the tooltip vertically */ + min-width: 440px; + max-width: 600px; + background-color: #333; + color: #fff; + padding: 8px; + border-radius: 4px; + font-size: 0.8em; + text-align: left; + white-space: normal; + z-index: 10; + } + + .info-btn:hover .tooltip-text { + display: block; + } + `; +} + +customElements.define('mk20-ddo-deal-list', MK20DDODealList); \ No newline at end of file diff --git a/web/static/pages/mk20/index.html b/web/static/pages/mk20/index.html new file mode 100644 index 000000000..992315996 --- /dev/null +++ b/web/static/pages/mk20/index.html @@ -0,0 +1,25 @@ + + + + Storage Marker + + + + + + +
+
+
+

Storage Deals

+
+
+
+ +
+
+
+
+ + + diff --git a/web/static/pages/sector/sector-info.mjs b/web/static/pages/sector/sector-info.mjs index 4d6efe072..ca1cbdab9 100644 --- a/web/static/pages/sector/sector-info.mjs +++ b/web/static/pages/sector/sector-info.mjs @@ -130,7 +130,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement { ${(this.data.Pieces||[]).map(piece => html` ${piece.PieceIndex} - ${piece.PieceCid} + ${piece.PieceCid} ${piece.PieceSize} ${piece.DealID} ${piece.DataUrl} diff --git a/web/static/ux/curio-ux.mjs b/web/static/ux/curio-ux.mjs index 5846aeb29..d360b2e38 100644 --- a/web/static/ux/curio-ux.mjs +++ b/web/static/ux/curio-ux.mjs @@ -199,7 +199,15 @@ class CurioUX extends LitElement { - Storage Deals + MK12 + + +
  • + + + + + MK20
  • From a72ecba6532da408431eb049fb5a8f0469b52541 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 20 May 2025 18:35:55 +0400 Subject: [PATCH 06/55] mk20 GC --- cmd/sptool/toolbox_deal_client.go | 3 +- .../default-curio-configuration.md | 2 +- documentation/en/curio-cli/sptool.md | 54 +++++++++++++++++++ .../harmonydb/sql/20250505-market_mk20.sql | 5 +- lib/commcidv2/commcidv2.go | 3 +- market/mk20/info.md | 39 +++++++------- market/mk20/types.go | 3 +- market/mk20/utils.go | 2 +- tasks/gc/pipeline_meta_gc.go | 27 ++++++++-- tasks/indexing/task_indexing.go | 6 +-- tasks/indexing/task_ipni.go | 4 +- web/api/webrpc/deals.go | 7 +-- web/api/webrpc/ipni.go | 3 +- web/api/webrpc/market.go | 4 +- web/api/webrpc/market_20.go | 10 ++-- web/api/webrpc/sector.go | 2 +- 16 files changed, 124 insertions(+), 50 deletions(-) diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index bdd8993cd..2150d2182 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -19,7 +19,6 @@ import ( "time" "github.com/dustin/go-humanize" - "github.com/filecoin-project/curio/market/mk20" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/maurl" @@ -43,6 +42,8 @@ import ( "github.com/filecoin-project/curio/lib/keystore" mk12_libp2p "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/mk12" + "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/lotus/api" chain_types "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index aa07c553f..932a488fa 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -509,7 +509,7 @@ description: The default curio configuration # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "5m0s") # # type: time.Duration - #IdleTimeout = "2m0s" + #IdleTimeout = "1h0m0s" # ReadHeaderTimeout is amount of time allowed to read request headers # Time duration string (e.g., "1h2m3s") in TOML format. (Default: "5m0s") diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 06c6313fe..52d0e94f2 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -503,6 +503,7 @@ USAGE: COMMANDS: spark Manage Smart Contract PeerID used by Spark mk12-client mk12 client for Curio + mk20-client mk20 client for Curio help, h Shows a list of commands or help for one command OPTIONS: @@ -888,3 +889,56 @@ USAGE: OPTIONS: --help, -h show help ``` + +### sptool toolbox mk20-client +``` +NAME: + sptool toolbox mk20-client - mk20 client for Curio + +USAGE: + sptool toolbox mk20-client command [command options] + +COMMANDS: + init Initialise curio mk12 client repo + deal Make a mk20 deal with Curio + help, h Shows a list of commands or help for one command + +OPTIONS: + --mk12-client-repo value repo directory for mk12 client (default: "~/.curio-client") [$CURIO_MK12_CLIENT_REPO] + --help, -h show help +``` + +#### sptool toolbox mk20-client init +``` +NAME: + sptool toolbox mk20-client init - Initialise curio mk12 client repo + +USAGE: + sptool toolbox mk20-client init [command options] + +OPTIONS: + --help, -h show help +``` + +#### sptool toolbox mk20-client deal +``` +NAME: + sptool toolbox mk20-client deal - Make a mk20 deal with Curio + +USAGE: + sptool toolbox mk20-client deal [command options] + +OPTIONS: + --http-url value http url to CAR file + --http-headers value [ --http-headers value ] http headers to be passed with the request (e.g key=value) + --car-size value size of the CAR file: required for online deals (default: 0) + --provider value storage provider on-chain address + --commp value commp of the CAR file + --piece-size value size of the CAR file as a padded piece (default: 0) + --duration value duration of the deal in epochs (default: 518400) + --verified whether the deal funds should come from verified client data-cap (default: false) + --indexing indicates that an deal should be indexed (default: true) + --wallet value wallet address to be used to initiate the deal + --announce indicates that deal should be announced to the IPNI(Network Indexer) (default: true) + --help, -h show help +``` diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index b9590fe72..d1af07263 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -154,10 +154,7 @@ CREATE TABLE market_mk20_offline_urls ( url TEXT NOT NULL, headers jsonb NOT NULL DEFAULT '{}', raw_size BIGINT NOT NULL, - PRIMARY KEY (id, piece_cid, piece_size), - CONSTRAINT market_mk20_offline_urls_id_fk FOREIGN KEY (id) - REFERENCES market_mk20_pipeline (id) - ON DELETE CASCADE + PRIMARY KEY (id, piece_cid, piece_size) ); CREATE TABLE market_mk20_products ( diff --git a/lib/commcidv2/commcidv2.go b/lib/commcidv2/commcidv2.go index fd5b93f1d..38b131fc7 100644 --- a/lib/commcidv2/commcidv2.go +++ b/lib/commcidv2/commcidv2.go @@ -3,12 +3,13 @@ package commcidv2 import ( "math/bits" - filabi "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" pool "github.com/libp2p/go-buffer-pool" "github.com/multiformats/go-multihash" "github.com/multiformats/go-varint" "golang.org/x/xerrors" + + filabi "github.com/filecoin-project/go-state-types/abi" ) type CommP struct { diff --git a/market/mk20/info.md b/market/mk20/info.md index 19c57b3dd..e39d83d26 100644 --- a/market/mk20/info.md +++ b/market/mk20/info.md @@ -96,19 +96,19 @@ DataSource represents the source of piece data, including metadata and optional | Field | Type | Tag | Description | |-------|------|-----|-------------| -| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piececid" | PieceCID represents the unique identifier for a piece of data, stored as a CID object. | +| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | PieceCID represents the unique identifier for a piece of data, stored as a CID object. | | Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"size" | Size represents the size of the padded piece in the data source. | | Format | [mk20.PieceDataFormat](#piecedataformat) | json:"format" | Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. | -| SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"sourcehttp" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | -| SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"sourceaggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | -| SourceOffline | [*mk20.DataSourceOffline](#datasourceoffline) | json:"sourceoffline" | SourceOffline defines the data source for offline pieces, including raw size information. | -| SourceHttpPut | [*mk20.DataSourceHttpPut](#datasourcehttpput) | json:"sourcehttpput" | SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import | +| SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"source_http" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | +| SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"source_aggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | +| SourceOffline | [*mk20.DataSourceOffline](#datasourceoffline) | json:"source_offline" | SourceOffline defines the data source for offline pieces, including raw size information. | +| SourceHttpPut | [*mk20.DataSourceHttpPut](#datasourcehttpput) | json:"source_httpput" | SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import | ### Products | Field | Type | Tag | Description | |-------|------|-----|-------------| -| DDOV1 | [*mk20.DDOV1](#ddov1) | json:"ddov1" | DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) | +| DDOV1 | [*mk20.DDOV1](#ddov1) | json:"ddo_v1" | DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) | ### DDOV1 @@ -119,16 +119,16 @@ for a DDO deal handling. |-------|------|-----|-------------| | Provider | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"provider" | Provider specifies the address of the provider | | Client | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"client" | Client represents the address of the deal client | -| PieceManager | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"piecemanager" | Actor able to with AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs | +| PieceManager | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"piece_manager" | Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs | | Duration | [abi.ChainEpoch](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#ChainEpoch) | json:"duration" | Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400 | -| AllocationId | [*verifreg.AllocationId](https://pkg.go.dev/github.com/filecoin-project/go-state-types/builtin/v16/verifreg#AllocationId) | json:"aggregatedallocationid" | AllocationId represents an aggregated allocation identifier for the deal. | -| ContractAddress | [string](https://pkg.go.dev/builtin#string) | json:"contractaddress" | ContractAddress specifies the address of the contract governing the deal | -| ContractDealIDMethod | [string](https://pkg.go.dev/builtin#string) | json:"contractdealidmethod" | ContractDealIDMethod specifies the method name to retrieve the deal ID for a contract | -| ContractDealIDMethodParams | [[]byte](https://pkg.go.dev/builtin#byte) | json:"contractdealidmethodparams" | ContractDealIDMethodParams represents encoded parameters for the contract deal ID method if required by the contract | -| NotificationAddress | [string](https://pkg.go.dev/builtin#string) | json:"notificationaddress" | NotificationAddress specifies the address to which notifications will be relayed to when sector is activated | -| NotificationPayload | [[]byte](https://pkg.go.dev/builtin#byte) | json:"notificationpayload" | NotificationPayload holds the notification data typically in a serialized byte array format. | +| AllocationId | [*verifreg.AllocationId](https://pkg.go.dev/github.com/filecoin-project/go-state-types/builtin/v16/verifreg#AllocationId) | json:"allocation_id" | AllocationId represents an aggregated allocation identifier for the deal. | +| ContractAddress | [string](https://pkg.go.dev/builtin#string) | json:"contract_address" | ContractAddress specifies the address of the contract governing the deal | +| ContractVerifyMethod | [string](https://pkg.go.dev/builtin#string) | json:"contract_verify_method" | ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract | +| ContractVerifyMethodParams | [[]byte](https://pkg.go.dev/builtin#byte) | json:"contract_verify_method_params" | ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract | +| NotificationAddress | [string](https://pkg.go.dev/builtin#string) | json:"notification_address" | NotificationAddress specifies the address to which notifications will be relayed to when sector is activated | +| NotificationPayload | [[]byte](https://pkg.go.dev/builtin#byte) | json:"notification_payload" | NotificationPayload holds the notification data typically in a serialized byte array format. | | Indexing | [bool](https://pkg.go.dev/builtin#bool) | json:"indexing" | Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval | -| AnnounceToIPNI | [bool](https://pkg.go.dev/builtin#bool) | json:"announcetoinpni" | AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). | +| AnnounceToIPNI | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_to_ipni" | AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). | ### DataSourceAggregate @@ -153,7 +153,7 @@ DataSourceHttpPut represents a data source allowing clients to push piece data a | Field | Type | Tag | Description | |-------|------|-----|-------------| -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | RawSize specifies the raw size of the data in bytes. | ### DataSourceOffline @@ -161,7 +161,7 @@ DataSourceOffline represents the data source for offline pieces, including metad | Field | Type | Tag | Description | |-------|------|-----|-------------| -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | RawSize specifies the raw size of the data in bytes. | ### DealStatusResponse @@ -170,7 +170,7 @@ DealStatusResponse represents the response of a deal's status, including its cur | Field | Type | Tag | Description | |-------|------|-----|-------------| | State | [mk20.DealState](#constants-for-dealstate) | json:"status" | State indicates the current processing state of the deal as a DealState value. | -| ErrorMsg | [string](https://pkg.go.dev/builtin#string) | json:"errormsg" | ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. | +| ErrorMsg | [string](https://pkg.go.dev/builtin#string) | json:"error_msg" | ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. | ### FormatAggregate @@ -190,11 +190,10 @@ FormatBytes defines the raw byte representation of data as a format. ### FormatCar -FormatCar represents the CAR (Content Addressable aRchive) format with version metadata for piece data serialization. +FormatCar represents the CAR (Content Addressable archive) format for piece data serialization. | Field | Type | Tag | Description | |-------|------|-----|-------------| -| Version | [uint64](https://pkg.go.dev/builtin#uint64) | json:"version" | Version specifies the version of the CAR format used for piece data serialization. | ### HttpUrl @@ -203,7 +202,7 @@ HttpUrl represents an HTTP endpoint configuration for fetching piece data. | Field | Type | Tag | Description | |-------|------|-----|-------------| | URL | [string](https://pkg.go.dev/builtin#string) | json:"url" | URL specifies the HTTP endpoint where the piece data can be fetched. | -| HTTPHeaders | [http.Header](https://pkg.go.dev/net/http#Header) | json:"httpheaders" | HTTPHeaders represents the HTTP headers associated with the URL. | +| Headers | [http.Header](https://pkg.go.dev/net/http#Header) | json:"headers" | HTTPHeaders represents the HTTP headers associated with the URL. | | Priority | [uint64](https://pkg.go.dev/builtin#uint64) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | | Fallback | [bool](https://pkg.go.dev/builtin#bool) | json:"fallback" | Fallback indicates whether this URL serves as a fallback option when other URLs fail. | diff --git a/market/mk20/types.go b/market/mk20/types.go index a9f615c9d..319d58b2f 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -3,9 +3,10 @@ package mk20 import ( "net/http" - "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" "github.com/oklog/ulid" + + "github.com/filecoin-project/go-state-types/abi" ) // Deal represents a structure defining the details and components of a specific deal in the system. diff --git a/market/mk20/utils.go b/market/mk20/utils.go index b59856599..a7215b2c5 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -12,12 +12,12 @@ import ( "net/url" "time" - "github.com/filecoin-project/go-address" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-data-segment/datasegment" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index 9dc7269d4..0cf946987 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -38,6 +38,9 @@ func (s *PipelineGC) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("cleanupUnseal: %w", err) } + if err := s.cleanupMK20DealPipeline(); err != nil { + return false, xerrors.Errorf("cleanupMK20DealPipeline: %w", err) + } return true, nil } @@ -158,13 +161,9 @@ func (s *PipelineGC) cleanupUpgrade() error { } func (s *PipelineGC) cleanupMK12DealPipeline() error { - // Remove market_mk12_deal_pipeline entries where: - // sealed is true and indexed is true ctx := context.Background() - // Execute the query - // NOTE: pipelines can be complete before indexing finishes in case of reindexing pipeline tasks (created in CheckIndex task) - _, err := s.db.Exec(ctx, `DELETE FROM market_mk12_deal_pipeline WHERE (should_index = FALSE OR indexed = TRUE) AND complete = TRUE;`) + _, err := s.db.Exec(ctx, `DELETE FROM market_mk12_deal_pipeline WHERE complete = TRUE;`) if err != nil { return xerrors.Errorf("failed to clean up sealed deals: %w", err) } @@ -177,6 +176,24 @@ func (s *PipelineGC) cleanupMK12DealPipeline() error { return nil } +func (s *PipelineGC) cleanupMK20DealPipeline() error { + ctx := context.Background() + + _, err := s.db.Exec(ctx, `DELETE FROM market_mk20_offline_urls + WHERE id IN ( + SELECT id FROM market_mk20_pipeline WHERE complete = TRUE + ); + + DELETE FROM market_mk20_pipeline + WHERE complete = TRUE; + `) + if err != nil { + return xerrors.Errorf("failed to clean up sealed deals: %w", err) + } + + return nil +} + func (s *PipelineGC) cleanupUnseal() error { // Remove sectors_unseal_pipeline entries where: // after_unseal_sdr is true diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 68f5080ba..36efcbcb7 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -12,9 +12,6 @@ import ( "sync" "time" - "github.com/filecoin-project/curio/market/mk20" - "github.com/filecoin-project/go-data-segment/datasegment" - "github.com/filecoin-project/go-data-segment/fr32" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" carv2 "github.com/ipld/go-car/v2" @@ -23,6 +20,8 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" + "github.com/filecoin-project/go-data-segment/datasegment" + "github.com/filecoin-project/go-data-segment/fr32" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/deps/config" @@ -36,6 +35,7 @@ import ( "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/mk20" ) var log = logging.Logger("indexing") diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 3029e19fc..94f2f8136 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -11,8 +11,6 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/lib/cachedreader" - "github.com/filecoin-project/curio/market/mk20" "github.com/google/uuid" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -36,12 +34,14 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/cachedreader" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/ipni/chunker" "github.com/filecoin-project/curio/market/ipni/ipniculib" + "github.com/filecoin-project/curio/market/mk20" ) var ilog = logging.Logger("ipni") diff --git a/web/api/webrpc/deals.go b/web/api/webrpc/deals.go index 6b41c436a..7c7f31c33 100644 --- a/web/api/webrpc/deals.go +++ b/web/api/webrpc/deals.go @@ -4,12 +4,13 @@ import ( "context" "time" - "github.com/filecoin-project/curio/lib/commcidv2" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/storageingest" "github.com/filecoin-project/lotus/chain/types" diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index 3281cdaec..fe9c8a771 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -11,12 +11,13 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/lib/commcidv2" "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/commcidv2" ) type IpniAd struct { diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 3f173e536..b1e019c97 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -11,8 +11,6 @@ import ( "strings" "time" - "github.com/filecoin-project/curio/lib/commcidv2" - "github.com/filecoin-project/curio/market/mk20" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/samber/lo" @@ -25,6 +23,8 @@ import ( "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/market/mk20" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index 1f73c6dc4..7f93ce22b 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -5,13 +5,15 @@ import ( "database/sql" "fmt" - "github.com/filecoin-project/curio/lib/commcidv2" - "github.com/filecoin-project/curio/market/mk20" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/market/mk20" ) type MK20StorageDeal struct { diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index 0e851bbae..6829bf569 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -7,7 +7,6 @@ import ( "time" "github.com/docker/go-units" - "github.com/filecoin-project/curio/lib/commcidv2" "github.com/ipfs/go-cid" "github.com/samber/lo" "github.com/snadrus/must" @@ -17,6 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" From 8b68feaa97e68cacb511e0b9135885d706f50c4f Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 23 May 2025 23:43:32 +0400 Subject: [PATCH 07/55] aggregate deals --- Dockerfile | 8 +- Makefile | 2 +- cmd/curio/tasks/tasks.go | 3 + cmd/sptool/toolbox_deal_client.go | 210 +++++- cmd/sptool/toolbox_deal_tools.go | 3 +- .../piece-server/sample/mk20-aggregate-car.sh | 95 +++ docker/piece-server/sample/mk20-ddo.sh | 31 + .../piece-server/sample/mk20-random-deal.sh | 24 + go.mod | 1 + go.sum | 1 + .../harmonydb/sql/20250505-market_mk20.sql | 57 +- lib/testutils/testutils.go | 119 ++- market/indexstore/indexstore_test.go | 2 +- market/mk20/ddo_v1.go | 27 +- market/mk20/http/http.go | 93 ++- market/mk20/{ => http}/info.md | 0 market/mk20/http/test.html | 692 ++++++++++++++++++ market/mk20/mk20.go | 8 +- market/mk20/types.go | 2 +- market/mk20/utils.go | 37 +- tasks/indexing/task_check_indexes.go | 32 +- tasks/indexing/task_indexing.go | 168 +++-- tasks/indexing/task_ipni.go | 74 +- tasks/seal/finalize_pieces.go | 2 +- tasks/seal/task_movestorage.go | 14 +- tasks/snap/task_movestorage.go | 14 +- tasks/storage-market/market_balance.go | 19 +- tasks/storage-market/mk20.go | 342 +++++---- tasks/storage-market/storage_market.go | 5 +- tasks/storage-market/task_aggregation.go | 47 +- tasks/storage-market/task_commp.go | 92 +-- web/api/webrpc/ipni.go | 2 +- web/api/webrpc/market.go | 97 ++- web/api/webrpc/market_20.go | 71 +- web/static/pages/mk20-deal/deal.mjs | 65 +- web/static/pages/mk20-deal/index.html | 1 - web/static/pages/piece/piece-info.mjs | 176 ++++- 37 files changed, 2103 insertions(+), 533 deletions(-) create mode 100755 docker/piece-server/sample/mk20-aggregate-car.sh create mode 100755 docker/piece-server/sample/mk20-ddo.sh create mode 100755 docker/piece-server/sample/mk20-random-deal.sh rename market/mk20/{ => http}/info.md (100%) create mode 100644 market/mk20/http/test.html diff --git a/Dockerfile b/Dockerfile index 355d58d75..2e9f131a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,11 @@ ENV RUSTUP_HOME=/usr/local/rustup \ PATH=/usr/local/cargo/bin:$PATH \ RUST_VERSION=1.63.0 +COPY ./ /opt/curio +WORKDIR /opt/curio +RUN git submodule update --init +RUN go mod download + RUN set -eux; \ dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ @@ -32,9 +37,6 @@ RUN set -eux; \ cargo --version; \ rustc --version; -COPY ./ /opt/curio -WORKDIR /opt/curio - ### make configurable filecoin-ffi build ARG FFI_BUILD_FROM_SOURCE=0 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE} diff --git a/Makefile b/Makefile index 01f4260ea..1b7c1e3b1 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ gen: gensimple .PHONY: gen marketgen: - $(GOCC) run ./market/mk20/mk20gen -pkg ./market/mk20 -output ./market/mk20/info.md + $(GOCC) run ./market/mk20/mk20gen -pkg ./market/mk20 -output ./market/mk20/http/info.md .PHONY: marketgen gensimple: api-gen go-generate cfgdoc-gen docsgen marketgen docsgen-cli diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index b1a47579f..e0ffd9045 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -261,6 +261,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan activeTasks = append(activeTasks, commpTask) } + aggTask := storage_market.NewAggregateTask(dm, db, must.One(slrLazy.Val()), lstor, full) + activeTasks = append(activeTasks, aggTask) + // PSD and Deal find task do not require many resources. They can run on all machines psdTask := storage_market.NewPSDTask(dm, db, sender, as, &cfg.Market.StorageMarketConfig.MK12, full) dealFindTask := storage_market.NewFindDealTask(dm, db, full, &cfg.Market.StorageMarketConfig.MK12) diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 2150d2182..7fdd69c2d 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -9,11 +9,13 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "os" "os/signal" "path/filepath" + "strconv" "strings" "syscall" "time" @@ -21,6 +23,7 @@ import ( "github.com/dustin/go-humanize" "github.com/google/uuid" "github.com/ipfs/go-cid" + "github.com/ipfs/go-cidutil/cidenc" "github.com/ipni/go-libipni/maurl" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" @@ -29,6 +32,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" "github.com/urfave/cli/v2" "golang.org/x/term" "golang.org/x/xerrors" @@ -37,9 +41,11 @@ import ( cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/curio/lib/keystore" + "github.com/filecoin-project/curio/lib/testutils" mk12_libp2p "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/mk12" "github.com/filecoin-project/curio/market/mk20" @@ -1572,6 +1578,7 @@ var mk20Clientcmd = &cli.Command{ Subcommands: []*cli.Command{ initCmd, mk20DealCmd, + mk20ClientMakeAggregateCmd, }, } @@ -1580,18 +1587,16 @@ var mk20DealCmd = &cli.Command{ Usage: "Make a mk20 deal with Curio", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "http-url", - Usage: "http url to CAR file", - Required: true, + Name: "http-url", + Usage: "http url to CAR file", }, &cli.StringSliceFlag{ Name: "http-headers", Usage: "http headers to be passed with the request (e.g key=value)", }, &cli.Uint64Flag{ - Name: "car-size", - Usage: "size of the CAR file: required for online deals", - Required: true, + Name: "car-size", + Usage: "size of the CAR file: required for online deals", }, &cli.StringFlag{ Name: "provider", @@ -1613,10 +1618,19 @@ var mk20DealCmd = &cli.Command{ Usage: "duration of the deal in epochs", Value: 518400, // default is 2880 * 180 == 180 days }, - &cli.BoolFlag{ - Name: "verified", - Usage: "whether the deal funds should come from verified client data-cap", - Value: false, + &cli.StringFlag{ + Name: "contract-address", + Usage: "contract address of the deal", + Required: true, + }, + &cli.StringFlag{ + Name: "contract-verify-method", + Usage: "contract verify method of the deal", + Required: true, + }, + &cli.Uint64Flag{ + Name: "allocation", + Usage: "allocation id of the deal", }, &cli.BoolFlag{ Name: "indexing", @@ -1632,6 +1646,10 @@ var mk20DealCmd = &cli.Command{ Usage: "indicates that deal should be announced to the IPNI(Network Indexer)", Value: true, }, + &cli.StringFlag{ + Name: "aggregate", + Usage: "aggregate file path for the deal", + }, }, Action: func(cctx *cli.Context) error { ctx := cctx.Context @@ -1645,10 +1663,6 @@ var mk20DealCmd = &cli.Command{ return fmt.Errorf("cant setup gateway connection: %w", err) } defer closer() - if err != nil { - return xerrors.Errorf("cant setup gateway connection: %w", err) - } - defer closer() walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) if err != nil { @@ -1718,9 +1732,6 @@ var mk20DealCmd = &cli.Command{ } carFileSize := cctx.Uint64("car-size") - if carFileSize == 0 { - return xerrors.Errorf("size of car file cannot be 0") - } url, err := url.Parse(cctx.String("http-url")) if err != nil { @@ -1737,23 +1748,109 @@ var mk20DealCmd = &cli.Command{ headers.Add(sp[0], sp[1]) } - d := mk20.DataSource{ - PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceHTTP: &mk20.DataSourceHTTP{ - RawSize: carFileSize, - URLs: []mk20.HttpUrl{ - { - URL: url.String(), - Headers: headers, - Priority: 0, - Fallback: true, + var d mk20.DataSource + + if cctx.IsSet("aggregate") { + d = mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Aggregate: &mk20.FormatAggregate{ + Type: mk20.AggregateTypeV1, }, }, - }, + } + + var pieces []mk20.DataSource + + log.Debugw("using aggregate data source", "aggregate", cctx.String("aggregate")) + // Read file line by line + loc, err := homedir.Expand(cctx.String("aggregate")) + if err != nil { + return err + } + file, err := os.Open(loc) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Split(line, "\t") + if len(parts) != 4 { + return fmt.Errorf("invalid line format. Expected pieceCid, pieceSize, carSize, url at %s", line) + } + if parts[0] == "" || parts[1] == "" || parts[2] == "" || parts[3] == "" { + return fmt.Errorf("empty column value in the input file at %s", line) + } + + pieceCid, err := cid.Parse(parts[0]) + if err != nil { + return fmt.Errorf("failed to parse CID: %w", err) + } + pieceSize, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return fmt.Errorf("failed to parse size %w", err) + } + + rawSize, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return fmt.Errorf("failed to parse raw size %w", err) + } + + url, err := url.Parse(parts[3]) + if err != nil { + return fmt.Errorf("failed to parse url: %w", err) + } + + pieces = append(pieces, mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + RawSize: uint64(rawSize), + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Priority: 0, + Fallback: true, + }, + }, + }, + }) + + if err := scanner.Err(); err != nil { + return err + } + } + d.SourceAggregate = &mk20.DataSourceAggregate{ + Pieces: pieces, + } + } else { + if carFileSize == 0 { + return xerrors.Errorf("size of car file cannot be 0") + } + d = mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + RawSize: carFileSize, + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Headers: headers, + Priority: 0, + Fallback: true, + }, + }, + }, + } } p := mk20.Products{ @@ -1770,6 +1867,11 @@ var mk20DealCmd = &cli.Command{ }, } + if cctx.Uint64("allocation") != 0 { + alloc := verifreg.AllocationId(cctx.Uint64("allocation")) + p.DDOV1.AllocationId = &alloc + } + id, err := mk20.NewULID() if err != nil { return err @@ -1797,15 +1899,19 @@ var mk20DealCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to create request: %w", err) } - req.Header.Set("Content-Type", "application/cbor") + req.Header.Set("Content-Type", "application/json") + log.Debugw("Headers", "headers", req.Header) resp, err := http.DefaultClient.Do(req) if err != nil { log.Warnw("failed to send request", "url", s, "error", err) continue } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Warnw("failed to send request", "url", s, "status", resp.StatusCode, "body", resp.Body) + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + log.Warnw("failed to send request", "url", s, "status", resp.StatusCode, "body", string(respBody)) continue } return nil @@ -1813,3 +1919,37 @@ var mk20DealCmd = &cli.Command{ return xerrors.Errorf("failed to send request to any of the URLs") }, } + +var mk20ClientMakeAggregateCmd = &cli.Command{ + Name: "aggregate", + Usage: "Create a new aggregate from a list of CAR files", + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "files", + Usage: "list of CAR files to aggregate", + Required: true, + }, + &cli.Uint64Flag{ + Name: "piece-size", + Usage: "piece size of the aggregate", + Required: true, + }, + &cli.BoolFlag{ + Name: "out", + Usage: "output the aggregate file", + }, + }, + Action: func(cctx *cli.Context) error { + size := abi.PaddedPieceSize(cctx.Uint64("piece-size")) + files := cctx.StringSlice("files") + out := cctx.Bool("out") + pcid, size, err := testutils.CreateAggregateFromCars(files, size, out) + if err != nil { + return err + } + encoder := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} + fmt.Println("CommP CID: ", encoder.Encode(pcid)) + fmt.Println("Piece size: ", size) + return nil + }, +} diff --git a/cmd/sptool/toolbox_deal_tools.go b/cmd/sptool/toolbox_deal_tools.go index ddd4c8eed..4c9274a00 100644 --- a/cmd/sptool/toolbox_deal_tools.go +++ b/cmd/sptool/toolbox_deal_tools.go @@ -10,7 +10,6 @@ import ( "path" "strconv" "strings" - "time" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" @@ -286,7 +285,7 @@ var generateRandCar = &cli.Command{ cs := cctx.Int64("chunksize") ml := cctx.Int("maxlinks") - rf, err := testutils.CreateRandomFile(outPath, time.Now().Unix(), size) + rf, err := testutils.CreateRandomTmpFile(outPath, size) if err != nil { return err } diff --git a/docker/piece-server/sample/mk20-aggregate-car.sh b/docker/piece-server/sample/mk20-aggregate-car.sh new file mode 100755 index 000000000..e95f6d2bf --- /dev/null +++ b/docker/piece-server/sample/mk20-aggregate-car.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -e + +# ANSI escape codes for styling +ci="\e[3m" +cn="\e[0m" + +# Parameters for file generation +chunks=512 +links=8 +output_dir="/var/lib/curio-client/data/" +size=99700 +num_files=63 +piece_size=$((8 * 1024 * 1024)) # 8 MiB + +# Array to store generated CAR files +declare -a car_files + +# Step 1: Generate all files +echo "Generating $num_files random CAR files (size: $size bytes):" +for i in $(seq 1 "$num_files"); do + echo "Generating file $i..." + output=$(sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=$size "$output_dir" 2>&1) + car_file=$(echo "$output" | awk '{print $NF}') + new_car_file="${car_file%.car}" + mv "$car_file" "$new_car_file" + car_file="$new_car_file" + + if [[ -n "$car_file" ]]; then + car_files+=("$car_file") + echo "File $i generated: $car_file" + else + echo "Error: Failed to generate file $i" >&2 + exit 1 + fi +done + +if [[ ${#car_files[@]} -eq 0 ]]; then + echo "Error: No files were generated. Exiting." >&2 + exit 1 +fi + + +# Declare the base command and arguments +base_command="sptool --actor t01000 toolbox mk20-client aggregate --piece-size=$piece_size" + +# Append --file arguments for each file in the car_files array +for car_file in "${car_files[@]}"; do + base_command+=" --files=$car_file" +done + +# Debugging: Print the full constructed command +printf "${ci}%s\n\n${cn}" "$base_command" + +# Execute the constructed command +aggregate_output=$($base_command 2>&1) + +echo "$aggregate_output" + +# Step 3: Extract `CommP CID` and `Piece size` from the aggregate output +commp_cid=$(echo "$aggregate_output" | awk -F': ' '/CommP CID/ {print $2}' | xargs) +piece_size=$(echo "$aggregate_output" | awk -F': ' '/Piece size/ {print $2}' | xargs) + +# Validate that we got proper output +if [[ -z "$commp_cid" || -z "$piece_size" ]]; then + echo "Error: Failed to extract CommP CID or Piece size from aggregation output" >&2 + exit 1 +fi + +# Step 4: Check and display the aggregate file +aggregate_file="aggregate_${commp_cid}" +if [[ -f "$aggregate_file" ]]; then + echo "Aggregate file stored at: $aggregate_file" + echo "Content of $aggregate_file:" + cat "$aggregate_file" +else + echo "Error: Aggregate file $aggregate_file not found!" >&2 +fi + +# Step 5: Print Results +echo -e "\n${ci}Aggregation Results:${cn}" +echo "CommP CID: $commp_cid" +echo "Piece Size: $piece_size" + + +miner_actor=$(lotus state list-miners | grep -v t01000) + +################################################################################### +printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +--commp=$commp_cid --piece-size=$piece_size --contract-address 0xtest --contract-verify-method test \ +--aggregate "$aggregate_file"\n\n${cn}" + +sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$commp_cid --piece-size=$piece_size --contract-address 0xtest --contract-verify-method test --aggregate "$aggregate_file" + +echo -e "\nDone!" \ No newline at end of file diff --git a/docker/piece-server/sample/mk20-ddo.sh b/docker/piece-server/sample/mk20-ddo.sh new file mode 100755 index 000000000..406aaf78c --- /dev/null +++ b/docker/piece-server/sample/mk20-ddo.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -e + +ci="\e[3m" +cn="\e[0m" + +chunks="${1:-51200}" +links="${2:-100}" + +printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" + +FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` +PAYLOAD_CID=$(find "$FILE" | xargs -I{} basename {} | sed 's/\.car//') + +read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk12-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') +miner_actor=$(lotus state list-miners | grep -v t01000) + +mv /var/lib/curio-client/data/$PAYLOAD_CID.car /var/lib/curio-client/data/$COMMP_CID + +sptool --actor t01000 toolbox mk12-client allocate -y -p $miner_actor --piece-cid $COMMP_CID --piece-size $PIECE --confidence 0 + +CLIENT=$(sptool --actor t01000 toolbox mk12-client wallet default) + +ALLOC=$(sptool --actor t01000 toolbox mk12-client list-allocations -j | jq -r --arg cid "$COMMP_CID" '.allocations | to_entries[] | select(.value.Data["/"] == $cid) | .key') + +printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +--http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID \ +--commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ +--contract-address 0xtest --contract-verify-method test --allocation=$ALLOC\n\n${cn}" + +sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test --allocation $ALLOC \ No newline at end of file diff --git a/docker/piece-server/sample/mk20-random-deal.sh b/docker/piece-server/sample/mk20-random-deal.sh new file mode 100755 index 000000000..9bf11ce12 --- /dev/null +++ b/docker/piece-server/sample/mk20-random-deal.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +ci="\e[3m" +cn="\e[0m" + +chunks="${1:-51200}" +links="${2:-100}" + +printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" + +FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` +PAYLOAD_CID=$(find "$FILE" | xargs -I{} basename {} | sed 's/\.car//') + +read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk12-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') +miner_actor=$(lotus state list-miners | grep -v t01000) + +################################################################################### +printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +--http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID \ +--commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ +--contract-address 0xtest --contract-verify-method test\n\n${cn}" + +sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test \ No newline at end of file diff --git a/go.mod b/go.mod index 9e8e9533f..82c849419 100644 --- a/go.mod +++ b/go.mod @@ -101,6 +101,7 @@ require ( github.com/whyrusleeping/cbor-gen v0.3.1 github.com/yugabyte/gocql v1.6.0-yb-1 github.com/yugabyte/pgx/v5 v5.5.3-yb-2 + github.com/yuin/goldmark v1.4.13 go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 diff --git a/go.sum b/go.sum index 7c6765737..a00ac5eff 100644 --- a/go.sum +++ b/go.sum @@ -1444,6 +1444,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index d1af07263..cc53ed08c 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -10,6 +10,57 @@ DROP CONSTRAINT IF EXISTS market_piece_meta_identity_key; ALTER TABLE market_piece_metadata ADD PRIMARY KEY (piece_cid, piece_size); +-- Drop the current primary key for market_piece_deal +ALTER TABLE market_piece_deal +DROP CONSTRAINT market_piece_deal_pkey; + +-- Drop the old UNIQUE constraint for market_piece_deal +ALTER TABLE market_piece_deal +DROP CONSTRAINT IF EXISTS market_piece_deal_identity_key; + +-- Add the new composite primary key for market_piece_deal +ALTER TABLE market_piece_deal + ADD PRIMARY KEY (sp_id, id, piece_cid, piece_length); + + +-- This function is used to insert piece metadata and piece deal (piece indexing) +-- This makes it easy to keep the logic of how table is updated and fast (in DB). +CREATE OR REPLACE FUNCTION process_piece_deal( + _id TEXT, + _piece_cid TEXT, + _boost_deal BOOLEAN, + _sp_id BIGINT, + _sector_num BIGINT, + _piece_offset BIGINT, + _piece_length BIGINT, -- padded length + _raw_size BIGINT, + _indexed BOOLEAN, + _legacy_deal BOOLEAN DEFAULT FALSE, + _chain_deal_id BIGINT DEFAULT 0 +) + RETURNS VOID AS $$ +BEGIN + -- Insert or update the market_piece_metadata table + INSERT INTO market_piece_metadata (piece_cid, piece_size, indexed) + VALUES (_piece_cid, _piece_length, _indexed) + ON CONFLICT (piece_cid, piece_size) DO UPDATE SET + indexed = CASE + WHEN market_piece_metadata.indexed = FALSE THEN EXCLUDED.indexed + ELSE market_piece_metadata.indexed + END; + + -- Insert into the market_piece_deal table + INSERT INTO market_piece_deal ( + id, piece_cid, boost_deal, legacy_deal, chain_deal_id, + sp_id, sector_num, piece_offset, piece_length, raw_size + ) VALUES ( + _id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id, + _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size + ) ON CONFLICT (sp_id, id, piece_cid, piece_length) DO NOTHING; + +END; +$$ LANGUAGE plpgsql; + -- Add ID column to ipni_task table ALTER TABLE ipni_task ADD COLUMN id TEXT; @@ -74,7 +125,7 @@ CREATE TABLE market_mk20_deal ( id TEXT PRIMARY KEY, piece_cid TEXT NOT NULL, - size BIGINT NOT NULL, + piece_size BIGINT NOT NULL, format JSONB NOT NULL, source_http JSONB NOT NULL DEFAULT 'null', @@ -103,7 +154,7 @@ CREATE TABLE market_mk20_pipeline ( announce BOOLEAN NOT NULL, allocation_id BIGINT DEFAULT NULL, duration BIGINT NOT NULL, - piece_aggregation INT DEFAULT 0, + piece_aggregation INT NOT NULL DEFAULT 0, started BOOLEAN DEFAULT FALSE, @@ -112,7 +163,7 @@ CREATE TABLE market_mk20_pipeline ( commp_task_id BIGINT DEFAULT NULL, after_commp BOOLEAN DEFAULT FALSE, - deal_aggregation INT DEFAULT 0, + deal_aggregation INT NOT NULL DEFAULT 0, aggr_index BIGINT DEFAULT 0, agg_task_id BIGINT DEFAULT NULL, aggregated BOOLEAN DEFAULT FALSE, diff --git a/lib/testutils/testutils.go b/lib/testutils/testutils.go index a8682f27b..30b6f8e3e 100644 --- a/lib/testutils/testutils.go +++ b/lib/testutils/testutils.go @@ -2,10 +2,13 @@ package testutils import ( "context" + "crypto/rand" "fmt" "io" - "math/rand" + "math/bits" "os" + "path" + "strings" "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" @@ -23,12 +26,18 @@ import ( carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" "github.com/multiformats/go-multihash" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-data-segment/datasegment" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" ) const defaultHashFunction = uint64(multihash.BLAKE2B_MIN + 31) -func CreateRandomFile(dir string, rseed int64, size int64) (string, error) { - source := io.LimitReader(rand.New(rand.NewSource(rseed)), size) +func CreateRandomTmpFile(dir string, size int64) (string, error) { + source := io.LimitReader(rand.Reader, size) file, err := os.CreateTemp(dir, "sourcefile.dat") if err != nil { @@ -162,3 +171,107 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, return nd.Cid(), nil } + +func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggregateOut bool) (cid.Cid, abi.PaddedPieceSize, error) { + var lines []string + var readers []io.Reader + var deals []abi.PieceInfo + + for _, f := range files { + file, err := os.Open(f) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("opening subpiece file: %w", err) + } + stat, err := file.Stat() + if err != nil { + return cid.Undef, 0, xerrors.Errorf("getting file stat: %w", err) + } + cp := new(commp.Calc) + _, err = io.Copy(cp, file) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("copying subpiece to commp writer: %w", err) + } + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("seeking to start of file: %w", err) + } + pbytes, size, err := cp.Digest() + if err != nil { + return cid.Undef, 0, xerrors.Errorf("computing digest for subpiece: %w", err) + } + pcid, err := commcid.DataCommitmentV1ToCID(pbytes) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("converting data commitment to CID: %w", err) + } + deals = append(deals, abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(size), + }) + readers = append(readers, file) + urlStr := fmt.Sprintf("http://piece-server:12320/pieces?id=%s", stat.Name()) + lines = append(lines, fmt.Sprintf("%s\t%d\t%d\t%s", pcid.String(), size, stat.Size(), urlStr)) + } + + _, upsize, err := datasegment.ComputeDealPlacement(deals) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("computing deal placement: %w", err) + } + + next := 1 << (64 - bits.LeadingZeros64(upsize+256)) + + if abi.PaddedPieceSize(next) != dealSize { + return cid.Undef, 0, fmt.Errorf("deal size mismatch: expected %d, got %d", dealSize, abi.PaddedPieceSize(next)) + } + + a, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), deals) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("creating aggregate: %w", err) + } + out, err := a.AggregateObjectReader(readers) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("creating aggregate reader: %w", err) + } + + p := path.Dir(files[0]) + + f, err := os.CreateTemp(p, "aggregate_*") + if err != nil { + return cid.Undef, 0, err + } + defer f.Close() + + cp := new(commp.Calc) + w := io.MultiWriter(cp, f) + + _, err = io.Copy(w, out) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("writing aggregate: %w", err) + } + + digest, paddedPieceSize, err := cp.Digest() + if err != nil { + return cid.Undef, 0, xerrors.Errorf("computing digest: %w", err) + } + if abi.PaddedPieceSize(paddedPieceSize) != dealSize { + return cid.Undef, 0, fmt.Errorf("deal size mismatch after final commP: expected %d, got %d", dealSize, abi.PaddedPieceSize(paddedPieceSize)) + } + + pcid, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("converting digest to CID: %w", err) + } + + err = os.WriteFile(fmt.Sprintf("aggregate_%s", pcid.String()), []byte(strings.Join(lines, "\n")), 0644) + if err != nil { + return cid.Undef, 0, xerrors.Errorf("writing aggregate to file: %w", err) + } + + if !aggregateOut { + defer os.Remove(f.Name()) + } else { + cn := path.Join(p, pcid.String()) + defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", cn)) + } + + return pcid, abi.PaddedPieceSize(paddedPieceSize), nil +} diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index af776f302..66b6424c0 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -42,7 +42,7 @@ func TestNewIndexStore(t *testing.T) { _ = os.RemoveAll(dir) }() - rf, err := testutils.CreateRandomFile(dir, time.Now().Unix(), 8000000) + rf, err := testutils.CreateRandomTmpFile(dir, time.Now().Unix(), 8000000) require.NoError(t, err) caropts := []carv2.Option{ diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index bee914785..e632c7bc1 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -2,8 +2,10 @@ package mk20 import ( "context" + "crypto/rand" "errors" "fmt" + "math/big" "net/http" "strings" @@ -11,6 +13,7 @@ import ( eabi "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" + "github.com/samber/lo" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" @@ -18,6 +21,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" ) @@ -65,7 +69,7 @@ type DDOV1 struct { AnnounceToIPNI bool `json:"announce_to_ipni"` } -func (d *DDOV1) Validate(db *harmonydb.DB) (ErrorCode, error) { +func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { code, err := IsProductEnabled(db, d.ProductName()) if err != nil { return code, err @@ -75,6 +79,19 @@ func (d *DDOV1) Validate(db *harmonydb.DB) (ErrorCode, error) { return ErrProductValidationFailed, xerrors.Errorf("provider address is not set") } + var mk20disabledMiners []address.Address + for _, m := range cfg.DisabledMiners { + maddr, err := address.NewFromString(m) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("failed to parse miner string: %s", err) + } + mk20disabledMiners = append(mk20disabledMiners, maddr) + } + + if lo.Contains(mk20disabledMiners, d.Provider) { + return ErrProductValidationFailed, xerrors.Errorf("provider is disabled") + } + if d.Client == address.Undef || d.Client.Empty() { return ErrProductValidationFailed, xerrors.Errorf("client address is not set") } @@ -119,6 +136,14 @@ func (d *DDOV1) Validate(db *harmonydb.DB) (ErrorCode, error) { } func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, ErrorCode, error) { + if d.ContractAddress == "0xtest" { + v, err := rand.Int(rand.Reader, big.NewInt(10000000)) + if err != nil { + return "", http.StatusInternalServerError, xerrors.Errorf("failed to generate random number: %w", err) + } + return v.String(), Ok, nil + } + var abiStr string err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) if err != nil { diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index ea05ddf1c..d5f9ada79 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -1,12 +1,13 @@ package http import ( + "bytes" "context" + _ "embed" "encoding/json" "fmt" "io" "net/http" - "os" "strings" "time" @@ -14,6 +15,10 @@ import ( "github.com/go-chi/httprate" logging "github.com/ipfs/go-log/v2" "github.com/oklog/ulid" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer/html" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -24,6 +29,9 @@ import ( storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) +//go:embed info.md +var infoMarkdown []byte + var log = logging.Logger("mk20httphdlr") const maxPutBodySize int64 = 64 << 30 // 64 GiB @@ -74,7 +82,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { var deal mk20.Deal if ct != "application/json" { log.Errorf("invalid content type: %s", ct) - w.WriteHeader(http.StatusBadRequest) + http.Error(w, "invalid content type", http.StatusBadRequest) return } @@ -82,15 +90,21 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { body, err := io.ReadAll(r.Body) if err != nil { log.Errorf("error reading request body: %s", err) - w.WriteHeader(http.StatusBadRequest) + http.Error(w, err.Error(), http.StatusBadRequest) + return } + + log.Infow("received deal proposal", "body", string(body)) + err = json.Unmarshal(body, &deal) if err != nil { log.Errorf("error unmarshaling json: %s", err) - w.WriteHeader(http.StatusBadRequest) + http.Error(w, err.Error(), http.StatusBadRequest) return } + log.Infow("received deal proposal", "deal", deal) + result := mdh.dm.MK20Handler.ExecuteDeal(context.Background(), &deal) log.Infow("deal processed", @@ -111,13 +125,13 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { log.Errorw("missing id in url", "url", r.URL) - w.WriteHeader(http.StatusBadRequest) + http.Error(w, "missing id in url", http.StatusBadRequest) return } id, err := ulid.Parse(idStr) if err != nil { log.Errorw("invalid id in url", "id", idStr, "err", err) - w.WriteHeader(http.StatusBadRequest) + http.Error(w, "invalid id in url", http.StatusBadRequest) return } @@ -202,16 +216,69 @@ func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Re // info serves the contents of the info file as a text/markdown response with HTTP 200 or returns an HTTP 500 on read/write failure. func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { - // Read the info File - data, err := os.ReadFile("../info.md") - if err != nil { - log.Errorw("failed to read info file", "err", err) - w.WriteHeader(http.StatusInternalServerError) + + var mdRenderer = goldmark.New( + goldmark.WithExtensions( + extension.GFM, + extension.Linkify, + extension.Table, + extension.DefinitionList, + ), + goldmark.WithRendererOptions( + html.WithHardWraps(), + html.WithXHTML(), + ), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + ), + ) + + var buf bytes.Buffer + if err := mdRenderer.Convert(infoMarkdown, &buf); err != nil { + http.Error(w, "failed to render markdown", http.StatusInternalServerError) return } + //if err := goldmark.Convert(infoMarkdown, &buf); err != nil { + // http.Error(w, "failed to render markdown", http.StatusInternalServerError) + // return + //} + w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/markdown") - _, err = w.Write(data) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + rendered := strings.ReplaceAll(buf.String(), "", `
    `) + + htmlStr := fmt.Sprintf(` + + + + + Curio Deal Schema + + + + + +
    +%s +
    + +`, rendered) + + _, err := w.Write([]byte(htmlStr)) if err != nil { log.Errorw("failed to write info file", "err", err) } diff --git a/market/mk20/info.md b/market/mk20/http/info.md similarity index 100% rename from market/mk20/info.md rename to market/mk20/http/info.md diff --git a/market/mk20/http/test.html b/market/mk20/http/test.html new file mode 100644 index 000000000..44a71201e --- /dev/null +++ b/market/mk20/http/test.html @@ -0,0 +1,692 @@ + + + + + + + Curio Deal Schema + + + + + +
    +

    Storage Market Interface

    +

    This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider.

    +

    šŸ“” MK20 HTTP API Overview

    +

    The MK20 storage market module provides a set of HTTP endpoints under /market/mk20 that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior.

    +

    Base URL

    +

    The base URL for all MK20 endpoints is:

    + 
    
+/market/mk20
+
+
    +

    šŸ”„ POST /store

    +

    Submit a new MK20 deal.

    +
      +
    • Content-Type: N/A
      • +
    • Body: N/A
      • +
    • Query Parameters: N/A
      • +
    • Response: +
        +
      • 200 OK: Deal accepted
        • +
      • Other HTTP codes indicate validation failure, rejection, or system errors
        • +
      +
      • +
    +

    šŸ§¾ GET /status?id=

    +

    Retrieve the current status of a deal.

    +
      +
    • Content-Type: application/json
      • +
    • Body: N/A
      • +
    • Query Parameters: +
        +
      • id: Deal identifier in ULID format
        • +
      +
      • +
    • Response: +
        +
      • 200 OK: JSON-encoded deal status information
        • +
      • 400 Bad Request: Missing or invalid ID
        • +
      • 500 Internal Server Error: If backend fails to respond
        • +
      +
      • +
    +

    šŸ“œ GET /contracts

    +
      +
    • +

      Content-Type: N/A

      +
      • +
    • +

      Body: N/A

      +
      • +
    • +

      Query Parameters: N/A
      + Return the list of contract addresses supported by the provider.

      +
      • +
    • +

      Response:

      + +
      • +
    +

    šŸ—‚ PUT /data?id=

    +

    Upload deal data after the deal has been accepted.

    +
      +
    • Content-Type: application/octet-stream
      • +
    • Body: Deal data bytes
      • +
    • Query Parameter:
      + -id: Deal identifier in ULID format
      • +
    • Headers: +
        +
      • Content-Length: must be deal's raw size
        • +
      +
      • +
    • Response: +
        +
      • 200 OK: if data is successfully streamed
        • +
      • 400, 413, or 415: on validation failures
        • +
      +
      • +
    +

    šŸ§  GET /info

    +
      +
    • +

      Content-Type: N/A

      +
      • +
    • +

      Body: N/A

      +
      • +
    • +

      Query Parameters: N/A
      + Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources.

      +
      • +
    • +

      Response:

      +
        +
      • 200 OK: with markdown content of the info file
        • +
      • 500 Internal Server Error: if file is not found or cannot be read
        • +
      +
      • +
    +

    Supported Deal Types

    +

    This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.

    +

    Deal

    +

    Deal represents a structure defining the details and components of a specific deal in the system.

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Identifierulid.ULIDjson:"identifier"Identifier represents a unique identifier for the deal in UUID format.
    Datamk20.DataSourcejson:"data"Data represents the source of piece data and associated metadata.
    Productsmk20.Productsjson:"products"Products represents a collection of product-specific information associated with a deal
    +

    DataSource

    +

    DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    PieceCIDcid.Cidjson:"piece_cid"PieceCID represents the unique identifier for a piece of data, stored as a CID object.
    Sizeabi.PaddedPieceSizejson:"size"Size represents the size of the padded piece in the data source.
    Formatmk20.PieceDataFormatjson:"format"Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats.
    SourceHTTP*mk20.DataSourceHTTPjson:"source_http"SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.
    SourceAggregate*mk20.DataSourceAggregatejson:"source_aggregate"SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.
    SourceOffline*mk20.DataSourceOfflinejson:"source_offline"SourceOffline defines the data source for offline pieces, including raw size information.
    SourceHttpPut*mk20.DataSourceHttpPutjson:"source_httpput"SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import
    +

    Products

    + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    DDOV1*mk20.DDOV1json:"ddo_v1"DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)
    +

    DDOV1

    +

    DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details
    + for a DDO deal handling.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Provideraddress.Addressjson:"provider"Provider specifies the address of the provider
    Clientaddress.Addressjson:"client"Client represents the address of the deal client
    PieceManageraddress.Addressjson:"piece_manager"Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs
    Durationabi.ChainEpochjson:"duration"Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400
    AllocationId*verifreg.AllocationIdjson:"allocation_id"AllocationId represents an aggregated allocation identifier for the deal.
    ContractAddressstringjson:"contract_address"ContractAddress specifies the address of the contract governing the deal
    ContractVerifyMethodstringjson:"contract_verify_method"ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract
    ContractVerifyMethodParams[]bytejson:"contract_verify_method_params"ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract
    NotificationAddressstringjson:"notification_address"NotificationAddress specifies the address to which notifications will be relayed to when sector is activated
    NotificationPayload[]bytejson:"notification_payload"NotificationPayload holds the notification data typically in a serialized byte array format.
    Indexingbooljson:"indexing"Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval
    AnnounceToIPNIbooljson:"announce_to_ipni"AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI).
    +

    DataSourceAggregate

    +

    DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces.

    + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Pieces[]mk20.DataSourcejson:"pieces"
    +

    DataSourceHTTP

    +

    DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs.

    + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    RawSizeuint64json:"rawsize"RawSize specifies the raw size of the data in bytes.
    URLs[]mk20.HttpUrljson:"urls"URLs lists the HTTP endpoints where the piece data can be fetched.
    +

    DataSourceHttpPut

    +

    DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted.

    + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    RawSizeuint64json:"raw_size"RawSize specifies the raw size of the data in bytes.
    +

    DataSourceOffline

    +

    DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece.

    + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    RawSizeuint64json:"raw_size"RawSize specifies the raw size of the data in bytes.
    +

    DealStatusResponse

    +

    DealStatusResponse represents the response of a deal's status, including its current state and an optional error message.

    + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Statemk20.DealStatejson:"status"State indicates the current processing state of the deal as a DealState value.
    ErrorMsgstringjson:"error_msg"ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.
    +

    FormatAggregate

    +

    FormatAggregate represents the aggregated format for piece data, identified by its type.

    + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Typemk20.AggregateTypejson:"type"Type specifies the type of aggregation for data pieces, represented by an AggregateType value.
    Sub[]mk20.PieceDataFormatjson:"sub"Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate
    +

    FormatBytes

    +

    FormatBytes defines the raw byte representation of data as a format.

    + + + + + + + + + +
    FieldTypeTagDescription
    +

    FormatCar

    +

    FormatCar represents the CAR (Content Addressable archive) format for piece data serialization.

    + + + + + + + + + +
    FieldTypeTagDescription
    +

    HttpUrl

    +

    HttpUrl represents an HTTP endpoint configuration for fetching piece data.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    URLstringjson:"url"URL specifies the HTTP endpoint where the piece data can be fetched.
    Headershttp.Headerjson:"headers"HTTPHeaders represents the HTTP headers associated with the URL.
    Priorityuint64json:"priority"Priority indicates the order preference for using the URL in requests, with lower values having higher priority.
    Fallbackbooljson:"fallback"Fallback indicates whether this URL serves as a fallback option when other URLs fail.
    +

    PieceDataFormat

    +

    PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Car*mk20.FormatCarjson:"car"Car represents the optional CAR file format, including its metadata and versioning details.
    Aggregate*mk20.FormatAggregatejson:"aggregate"Aggregate holds a reference to the aggregated format of piece data.
    Raw*mk20.FormatBytesjson:"raw"Raw represents the raw format of the piece data, encapsulated as bytes.
    +

    SupportedContracts

    +

    SupportedContracts represents a collection of contract addresses supported by a system or application.

    + + + + + + + + + + + + + + + + + +
    FieldTypeTagDescription
    Contracts[]stringjson:"contracts"Contracts represents a list of supported contract addresses in string format.
    +

    Constants for ErrorCode

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ConstantCodeDescription
    Ok200Ok represents a successful operation with an HTTP status code of 200.
    ErrBadProposal400ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic.
    ErrMalformedDataSource430ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data.
    ErrUnsupportedDataSource422ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context.
    ErrUnsupportedProduct423ErrUnsupportedProduct indicates that the requested product is not supported by the provider.
    ErrProductNotEnabled424ErrProductNotEnabled indicates that the requested product is not enabled on the provider.
    ErrProductValidationFailed425ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data.
    ErrDealRejectedByMarket426ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules.
    ErrServiceMaintenance503ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503.
    ErrServiceOverloaded429ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment.
    ErrMarketNotEnabled440ErrMarketNotEnabled indicates that the market is not enabled for the requested operation.
    ErrDurationTooShort441ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold.
    +

    Constants for DealState

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ConstantCodeDescription
    DealStateAccepted"accepted"DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system.
    DealStateProcessing"processing"DealStateProcessing represents the state of a deal currently being processed in the pipeline.
    DealStateSealing"sealing"DealStateSealing indicates that the deal is currently being sealed in the system.
    DealStateIndexing"indexing"DealStateIndexing represents the state where a deal is undergoing indexing in the system.
    DealStateFailed"failed"DealStateFailed indicates that the deal has failed due to an error during processing, sealing, or indexing.
    DealStateComplete"complete"DealStateComplete indicates that the deal has successfully completed all processing and is finalized in the system.
    + + + + \ No newline at end of file diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index f06f22e31..b8b153325 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -74,8 +74,8 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { - // Validate the DataSource TODO: Add error code to validate - code, err := deal.Validate(m.db) + // Validate the DataSource + code, err := deal.Validate(m.db, &m.cfg.Market.StorageMarketConfig.MK20) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ @@ -158,7 +158,9 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje } } - return nil + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusOK, + } } func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { diff --git a/market/mk20/types.go b/market/mk20/types.go index 319d58b2f..5d3bb6707 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -34,7 +34,7 @@ type DataSource struct { PieceCID cid.Cid `json:"piece_cid"` // Size represents the size of the padded piece in the data source. - Size abi.PaddedPieceSize `json:"size"` + Size abi.PaddedPieceSize `json:"piece_size"` // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` diff --git a/market/mk20/utils.go b/market/mk20/utils.go index a7215b2c5..17c67747f 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -22,11 +22,12 @@ import ( "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" ) -func (d *Deal) Validate(db *harmonydb.DB) (ErrorCode, error) { - code, err := d.Products.Validate(db) +func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { + code, err := d.Products.Validate(db, cfg) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) } @@ -261,19 +262,19 @@ func (d DataSource) RawSize() (uint64, error) { return 0, xerrors.Errorf("no source defined") } -func (d Products) Validate(db *harmonydb.DB) (ErrorCode, error) { +func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { if d.DDOV1 == nil { return ErrBadProposal, xerrors.Errorf("no products") } - return d.DDOV1.Validate(db) + return d.DDOV1.Validate(db, cfg) } type DBDeal struct { Identifier string `db:"id"` SpID int64 `db:"sp_id"` PieceCID string `db:"piece_cid"` - Size int64 `db:"size"` + Size int64 `db:"piece_size"` Format json.RawMessage `db:"format"` SourceHTTP json.RawMessage `db:"source_http"` SourceAggregate json.RawMessage `db:"source_aggregate"` @@ -382,7 +383,7 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } - n, err := tx.Exec(`INSERT INTO deals (id, sp_id, piece_cid, size, format, source_http, source_aggregate, source_offline, source_http_put, ddo_v1) + n, err := tx.Exec(`INSERT INTO market_mk20_deal (id, sp_id, piece_cid, piece_size, format, source_http, source_aggregate, source_offline, source_http_put, ddo_v1) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, dbDeal.Identifier, dbDeal.SpID, @@ -405,7 +406,17 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal - err := tx.Select(&dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) + err := tx.Select(&dbDeal, `SELECT + id, + piece_cid, + piece_size, + format, + source_http, + source_aggregate, + source_offline, + source_http_put, + ddo_v1, + error FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -417,7 +428,17 @@ func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { func DealFromDB(ctx context.Context, db *harmonydb.DB, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal - err := db.Select(ctx, &dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) + err := db.Select(ctx, &dbDeal, `SELECT + id, + piece_cid, + piece_size, + format, + source_http, + source_aggregate, + source_offline, + source_http_put, + ddo_v1, + error FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index eb47ecdfe..178860316 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" ) @@ -75,7 +76,7 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask err := c.db.Select(ctx, &toCheckList, ` SELECT mm.piece_cid, mpd.piece_length, mpd.piece_offset, mpd.sp_id, mpd.sector_num, mpd.raw_size FROM market_piece_metadata mm - LEFT JOIN market_piece_deal mpd ON mm.piece_cid = mpd.piece_cid + LEFT JOIN market_piece_deal mpd ON mm.piece_cid = mpd.piece_cid AND mm.piece_size = mpd.piece_length WHERE mm.indexed = true `) if err != nil { @@ -105,11 +106,23 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask var have, missing int64 for p, cent := range toCheck { - pieceCid, err := cid.Parse(p) + pCid, err := cid.Parse(p) if err != nil { return xerrors.Errorf("parsing piece cid: %w", err) } + pi := abi.PieceInfo{ + PieceCID: pCid, + Size: abi.PaddedPieceSize(cent[0].PieceLen), + } + + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return xerrors.Errorf("getting piece commP: %w", err) + } + + pieceCid := commp.PCidV2() + // Check if the piece is present in the index store hasEnt, err := c.indexStore.CheckHasPiece(ctx, pieceCid) if err != nil { @@ -131,8 +144,8 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask err = c.db.Select(ctx, &uuids, ` SELECT uuid FROM market_mk12_deals - WHERE piece_cid = $1 - `, pieceCid.String()) + WHERE piece_cid = $1 AND piece_size = $2 + `, pCid.String(), pi.Size) if err != nil { return xerrors.Errorf("getting deal uuids: %w", err) } @@ -260,8 +273,9 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.TaskID) (err error) { type pieceSP struct { - PieceCid string `db:"piece_cid"` - SpID int64 `db:"sp_id"` + PieceCid string `db:"piece_cid"` + PieceSize abi.PaddedPieceSize `db:"piece_size"` + SpID int64 `db:"sp_id"` } // get candidates to check @@ -301,7 +315,7 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas // get already running pipelines with announce=true var announcePiecePipelines []pieceSP - err = c.db.Select(ctx, &announcePiecePipelines, `SELECT piece_cid, sp_id FROM market_mk12_deal_pipeline WHERE announce=true`) + err = c.db.Select(ctx, &announcePiecePipelines, `SELECT piece_cid, piece_size, sp_id FROM market_mk12_deal_pipeline WHERE announce=true`) if err != nil { return xerrors.Errorf("getting ipni tasks: %w", err) } @@ -328,7 +342,7 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas }() for _, deal := range toCheck { - if _, ok := announcablePipelines[pieceSP{deal.PieceCID, deal.SpID}]; ok { + if _, ok := announcablePipelines[pieceSP{deal.PieceCID, deal.PieceSize, deal.SpID}]; ok { // pipeline for the piece already running have++ continue @@ -388,7 +402,7 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas PieceOffset int64 `db:"piece_offset"` RawSize int64 `db:"raw_size"` } - err = c.db.Select(ctx, &sourceSector, `SELECT sector_num, piece_offset, raw_size FROM market_piece_deal WHERE piece_cid=$1 AND sp_id = $2`, deal.PieceCID, deal.SpID) + err = c.db.Select(ctx, &sourceSector, `SELECT sector_num, piece_offset, raw_size FROM market_piece_deal WHERE piece_cid=$1 AND piece_length = $2 AND sp_id = $3`, deal.PieceCID, deal.PieceSize, deal.SpID) if err != nil { return xerrors.Errorf("getting source sector: %w", err) } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 36efcbcb7..d1b8e2b30 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "io" + "net/url" "runtime" "sort" + "strconv" "strings" "sync" "time" @@ -66,19 +68,20 @@ func NewIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore } type itask struct { - UUID string `db:"uuid"` - SpID int64 `db:"sp_id"` - Sector abi.SectorNumber `db:"sector"` - Proof abi.RegisteredSealProof `db:"reg_seal_proof"` - PieceCid string `db:"piece_cid"` - Size abi.PaddedPieceSize `db:"piece_size"` - Offset int64 `db:"sector_offset"` - RawSize int64 `db:"raw_size"` - ShouldIndex bool `db:"should_index"` - Announce bool `db:"announce"` - ChainDealId abi.DealID `db:"chain_deal_id"` - IsDDO bool `db:"is_ddo"` - Mk20 bool `db:"mk20"` + UUID string `db:"uuid"` + SpID int64 `db:"sp_id"` + Sector abi.SectorNumber `db:"sector"` + Proof abi.RegisteredSealProof `db:"reg_seal_proof"` + PieceCid string `db:"piece_cid"` + Size abi.PaddedPieceSize `db:"piece_size"` + Offset int64 `db:"sector_offset"` + RawSize int64 `db:"raw_size"` + ShouldIndex bool `db:"should_index"` + IndexingCreatedAt time.Time `db:"indexing_created_at"` + Announce bool `db:"announce"` + ChainDealId abi.DealID `db:"chain_deal_id"` + IsDDO bool `db:"is_ddo"` + Mk20 bool `db:"mk20"` } func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { @@ -100,7 +103,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do p.announce, p.is_ddo, COALESCE(d.chain_deal_id, 0) AS chain_deal_id, - false AS mk20 + FALSE AS mk20 FROM market_mk12_deal_pipeline p LEFT JOIN @@ -208,8 +211,6 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do dealCfg := i.cfg.Market.StorageMarketConfig chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize - opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} - recs := make(chan indexstore.Record, chanSize) var blocks int64 @@ -223,9 +224,9 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do }) if task.Mk20 && len(subPieces) > 0 { - blocks, interrupted, err = IndexAggregate(reader, task.Size, subPieces, opts, recs, addFail) + blocks, interrupted, err = IndexAggregate(reader, task.Size, subPieces, recs, addFail) } else { - blocks, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + blocks, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) } if err != nil { @@ -366,8 +367,8 @@ func validateSegments(segments []datasegment.SegmentDesc) []datasegment.SegmentD return validEntries } -func IndexCAR(r io.Reader, buffSize int, opts []carv2.Option, recs chan<- indexstore.Record, addFail <-chan struct{}) (int64, bool, error) { - blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(r, buffSize), opts...) +func IndexCAR(r io.Reader, buffSize int, recs chan<- indexstore.Record, addFail <-chan struct{}) (int64, bool, error) { + blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(r, buffSize), carv2.ZeroLengthSectionAsEOF(true)) if err != nil { return 0, false, fmt.Errorf("getting block reader over piece: %w", err) } @@ -414,7 +415,6 @@ func IndexAggregate( reader IndexReader, size abi.PaddedPieceSize, subPieces []mk20.PieceDataFormat, - opts []carv2.Option, recs chan<- indexstore.Record, addFail <-chan struct{}, ) (int64, bool, error) { @@ -436,6 +436,8 @@ func IndexAggregate( return 0, false, xerrors.New("no valid data segment index entries") } + log.Infow("Indexing aggregate", "piece_size", size, "num_chunks", len(valid), "num_sub_pieces", len(subPieces)) + var haveSubPieces bool if len(subPieces) > 0 { @@ -451,9 +453,11 @@ func IndexAggregate( if entry.Size < uint64(bufferSize) { bufferSize = int(entry.Size) } - sectionReader := io.NewSectionReader(reader, int64(entry.Offset), int64(entry.Size)) + strt := entry.UnpaddedOffest() + leng := entry.UnpaddedLength() + sectionReader := io.NewSectionReader(reader, int64(strt), int64(leng)) - b, inter, err := IndexCAR(sectionReader, bufferSize, opts, recs, addFail) + b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) totalBlocks += b if err != nil { @@ -466,7 +470,7 @@ func IndexAggregate( continue } if subPieces[j].Aggregate != nil { - b, inter, err = IndexAggregate(sectionReader, abi.PaddedPieceSize(entry.Size), nil, opts, recs, addFail) + b, inter, err = IndexAggregate(sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) if err != nil { return totalBlocks, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) } @@ -545,61 +549,94 @@ func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { ctx := context.Background() + type task struct { + TaskID harmonytask.TaskID `db:"indexing_task_id"` + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector"` + StorageID string `db:"storage_id"` + Url string `db:"url"` + Indexing bool `db:"indexing"` + } + + var tasks []*task + indIDs := make([]int64, len(ids)) for x, id := range ids { indIDs[x] = int64(id) } - // Accept any task which should not be indexed as - // it does not require storage access - var id int64 - err := i.db.QueryRow(ctx, `SELECT indexing_task_id - FROM market_mk12_deal_pipeline - WHERE should_index = FALSE - AND indexing_task_id = ANY ($1) - - UNION ALL - - SELECT indexing_task_id - FROM market_mk20_pipeline - WHERE indexing = FALSE - AND indexing_task_id = ANY ($1) - - ORDER BY indexing_task_id - LIMIT 1;`, indIDs).Scan(&id) - if err == nil { - ret := harmonytask.TaskID(id) - return &ret, nil - } else if !errors.Is(err, pgx.ErrNoRows) { - return nil, xerrors.Errorf("getting pending indexing task: %w", err) - } - - var tasks []struct { - TaskID harmonytask.TaskID `db:"indexing_task_id"` - SpID int64 `db:"sp_id"` - SectorNumber int64 `db:"sector"` - StorageID string `db:"storage_id"` + var mk20tasks []*task + if storiface.FTPiece != 32 { + panic("storiface.FTPiece != 32") + } + + err := i.db.Select(ctx, &mk20tasks, `SELECT indexing_task_id, url, indexing FROM market_mk20_pipeline WHERE indexing_task_id = ANY($1)`, indIDs) + if err != nil { + return nil, xerrors.Errorf("getting mk20 urls: %w", err) + } + + for _, t := range mk20tasks { + + if !t.Indexing { + continue + } + + goUrl, err := url.Parse(t.Url) + if err != nil { + return nil, xerrors.Errorf("parsing data URL: %w", err) + } + if goUrl.Scheme == "pieceref" { + refNum, err := strconv.ParseInt(goUrl.Opaque, 10, 64) + if err != nil { + return nil, xerrors.Errorf("parsing piece reference number: %w", err) + } + + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = i.db.Select(ctx, &pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, refNum) + if err != nil { + return nil, xerrors.Errorf("getting pieceID: %w", err) + } + + var sLocation string + + err = i.db.QueryRow(ctx, ` + SELECT storage_id FROM sector_location + WHERE miner_id = 0 AND sector_num = $1 AND sector_filetype = 32`, pieceID[0].PieceID).Scan(&sLocation) + + if err != nil { + return nil, xerrors.Errorf("failed to get storage location from DB: %w", err) + } + + t.StorageID = sLocation + + } } + log.Infow("mk20 tasks", "tasks", mk20tasks) + if storiface.FTUnsealed != 1 { panic("storiface.FTUnsealed != 1") } - err = i.db.Select(ctx, &tasks, `SELECT dp.indexing_task_id, dp.sp_id, dp.sector, l.storage_id + var mk12tasks []*task + + err = i.db.Select(ctx, &mk12tasks, `SELECT dp.indexing_task_id, dp.should_index AS indexing, dp.sp_id, dp.sector, l.storage_id FROM market_mk12_deal_pipeline dp INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num - WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1 - - UNION ALL - - SELECT dp.indexing_task_id, dp.sp_id, dp.sector, l.storage_id - FROM market_mk20_pipeline dp - INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num WHERE dp.indexing_task_id = ANY ($1) AND l.sector_filetype = 1`, indIDs) if err != nil { - return nil, xerrors.Errorf("getting tasks: %w", err) + return nil, xerrors.Errorf("getting mk12 tasks: %w", err) } + log.Infow("mk12 tasks", "tasks", mk12tasks) + + tasks = append(mk20tasks, mk12tasks...) + + log.Infow("tasks", "tasks", tasks) + ls, err := i.sc.LocalStorage(ctx) if err != nil { return nil, xerrors.Errorf("getting local storage: %w", err) @@ -611,6 +648,9 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T } for _, t := range tasks { + if !t.Indexing { + return &t.TaskID, nil + } if found, ok := localStorageMap[t.StorageID]; ok && found { return &t.TaskID, nil } @@ -710,7 +750,9 @@ func (i *IndexingTask) Adder(taskFunc harmonytask.AddTaskFunc) { func (i *IndexingTask) GetSpid(db *harmonydb.DB, taskID int64) string { var spid string - err := db.QueryRow(context.Background(), `SELECT sp_id FROM market_mk12_deal_pipeline WHERE indexing_task_id = $1`, taskID).Scan(&spid) + err := db.QueryRow(context.Background(), `SELECT sp_id FROM market_mk12_deal_pipeline WHERE indexing_task_id = $1 + UNION ALL + SELECT sp_id FROM market_mk20_pipeline WHERE indexing_task_id = $1`, taskID).Scan(&spid) if err != nil { log.Errorf("getting spid: %s", err) return "" diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 94f2f8136..da6dd8e61 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -14,7 +14,6 @@ import ( "github.com/google/uuid" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - carv2 "github.com/ipld/go-car/v2" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/maurl" @@ -35,9 +34,9 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" - "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/ipni/chunker" "github.com/filecoin-project/curio/market/ipni/ipniculib" @@ -137,8 +136,6 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b } } - opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} - recs := make(chan indexstore.Record, 1) var eg errgroup.Group @@ -147,6 +144,11 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b var subPieces []mk20.PieceDataFormat chk := chunker.NewInitialChunker() + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return false, xerrors.Errorf("getting piece commP: %w", err) + } + eg.Go(func() error { defer close(addFail) for rec := range recs { @@ -174,18 +176,18 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b } if deal.Data.Format.Car != nil { - _, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + _, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) } if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { subPieces = deal.Data.Format.Aggregate.Sub - _, interrupted, err = IndexAggregate(reader, pi.Size, subPieces, opts, recs, addFail) + _, interrupted, err = IndexAggregate(reader, pi.Size, subPieces, recs, addFail) } } } else { - _, interrupted, err = IndexCAR(reader, 4<<20, opts, recs, addFail) + _, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) } if err != nil { @@ -210,7 +212,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, nil } - lnk, err := chk.Finish(ctx, I.db, pi.PieceCID) + lnk, err := chk.Finish(ctx, I.db, commp.PCidV2()) if err != nil { return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) } @@ -331,57 +333,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b } func (I *IPNITask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { - var tasks []struct { - TaskID harmonytask.TaskID `db:"task_id"` - SpID int64 `db:"sp_id"` - SectorNumber int64 `db:"sector"` - StorageID string `db:"storage_id"` - } - - if storiface.FTUnsealed != 1 { - panic("storiface.FTUnsealed != 1") - } - - ctx := context.Background() - - indIDs := make([]int64, len(ids)) - for i, id := range ids { - indIDs[i] = int64(id) - } - - err := I.db.Select(ctx, &tasks, ` - SELECT dp.task_id, dp.sp_id, dp.sector, l.storage_id FROM ipni_task dp - INNER JOIN sector_location l ON dp.sp_id = l.miner_id AND dp.sector = l.sector_num - WHERE dp.task_id = ANY ($1) AND l.sector_filetype = 1 -`, indIDs) - if err != nil { - return nil, xerrors.Errorf("getting tasks: %w", err) - } - - ls, err := I.sc.LocalStorage(ctx) - if err != nil { - return nil, xerrors.Errorf("getting local storage: %w", err) - } - - acceptables := map[harmonytask.TaskID]bool{} - - for _, t := range ids { - acceptables[t] = true - } - - for _, t := range tasks { - if _, ok := acceptables[t.TaskID]; !ok { - continue - } - - for _, l := range ls { - if string(l.ID) == t.StorageID { - return &t.TaskID, nil - } - } - } - - return nil, nil + return &ids[0], nil } func (I *IPNITask) TypeDetails() harmonytask.TaskTypeDetails { @@ -429,6 +381,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun raw_size, should_index, announce, + indexing_created_at, FALSE as mk20 FROM market_mk12_deal_pipeline WHERE sealed = TRUE @@ -448,6 +401,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun raw_size, indexing AS should_index, announce, + indexing_created_at, TRUE as mk20 FROM market_mk20_pipeline WHERE sealed = TRUE @@ -544,7 +498,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun return false, xerrors.Errorf("marshaling piece info: %w", err) } - _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, p.SpID, p.UUID, + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, p.UUID, p.SpID, p.Sector, p.Proof, p.Offset, b.Bytes(), false, pid.String(), id) if err != nil { if harmonydb.IsErrUniqueContraint(err) { diff --git a/tasks/seal/finalize_pieces.go b/tasks/seal/finalize_pieces.go index de94df8e5..35f4d9939 100644 --- a/tasks/seal/finalize_pieces.go +++ b/tasks/seal/finalize_pieces.go @@ -38,7 +38,7 @@ func DropSectorPieceRefs(ctx context.Context, db *harmonydb.DB, sid abi.SectorID continue } - n, err := db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + n, err := db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1 AND long_term = FALSE`, refID) if err != nil { log.Errorw("failed to delete piece ref", "url", pu.URL, "error", err, "miner", sid.Miner, "sector", sid.Number) } diff --git a/tasks/seal/task_movestorage.go b/tasks/seal/task_movestorage.go index 3414b217b..3b8901f5d 100644 --- a/tasks/seal/task_movestorage.go +++ b/tasks/seal/task_movestorage.go @@ -68,17 +68,19 @@ func (m *MoveStorageTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) _, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { // Set indexing_created_at to Now() to allow new indexing tasks - _, err = tx.Exec(` - UPDATE market_mk20_pipeline + _, err = tx.Exec(`UPDATE market_mk20_pipeline SET indexing_created_at = NOW() - WHERE sp_id = $1 AND sector = $2; - - UPDATE market_mk12_deal_pipeline + WHERE sp_id = $1 AND sector = $2;`, task.SpID, task.SectorNumber) + if err != nil { + return false, fmt.Errorf("error creating indexing task for mk20 deals: %w", err) + } + + _, err = tx.Exec(`UPDATE market_mk12_deal_pipeline SET indexing_created_at = NOW() WHERE sp_id = $1 AND sector = $2; `, task.SpID, task.SectorNumber) if err != nil { - return false, fmt.Errorf("error creating indexing task: %w", err) + return false, fmt.Errorf("error creating indexing task for mk12: %w", err) } _, err = tx.Exec(`UPDATE sectors_sdr_pipeline SET after_move_storage = TRUE, task_id_move_storage = NULL WHERE task_id_move_storage = $1`, taskID) diff --git a/tasks/snap/task_movestorage.go b/tasks/snap/task_movestorage.go index 39e6c10a5..eece2e343 100644 --- a/tasks/snap/task_movestorage.go +++ b/tasks/snap/task_movestorage.go @@ -72,17 +72,19 @@ func (m *MoveStorageTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) _, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { // Set indexing_created_at to Now() to allow new indexing tasks - _, err = tx.Exec(` - UPDATE market_mk20_pipeline + _, err = tx.Exec(`UPDATE market_mk20_pipeline SET indexing_created_at = NOW() - WHERE sp_id = $1 AND sector = $2; - - UPDATE market_mk12_deal_pipeline + WHERE sp_id = $1 AND sector = $2;`, task.SpID, task.SectorNumber) + if err != nil { + return false, fmt.Errorf("error creating indexing task for mk20 deals: %w", err) + } + + _, err = tx.Exec(`UPDATE market_mk12_deal_pipeline SET indexing_created_at = NOW() WHERE sp_id = $1 AND sector = $2; `, task.SpID, task.SectorNumber) if err != nil { - return false, fmt.Errorf("error creating indexing task: %w", err) + return false, fmt.Errorf("error creating indexing task for mk12 deals: %w", err) } _, err = tx.Exec(`UPDATE sectors_snap_pipeline SET after_move_storage = TRUE, task_id_move_storage = NULL WHERE task_id_move_storage = $1`, taskID) diff --git a/tasks/storage-market/market_balance.go b/tasks/storage-market/market_balance.go index c31a5d6d0..96511ac83 100644 --- a/tasks/storage-market/market_balance.go +++ b/tasks/storage-market/market_balance.go @@ -43,20 +43,31 @@ type BalanceManager struct { } func NewBalanceManager(api mbalanceApi, miners []address.Address, cfg *config.CurioConfig, sender *message.Sender) (*BalanceManager, error) { - var disabledMiners []address.Address + var mk12disabledMiners []address.Address for _, m := range cfg.Market.StorageMarketConfig.MK12.DisabledMiners { maddr, err := address.NewFromString(m) if err != nil { return nil, xerrors.Errorf("failed to parse miner string: %s", err) } - disabledMiners = append(disabledMiners, maddr) + mk12disabledMiners = append(mk12disabledMiners, maddr) } - enabled, _ := lo.Difference(miners, disabledMiners) + mk12enabled, _ := lo.Difference(miners, mk12disabledMiners) + + var mk20disabledMiners []address.Address + for _, m := range cfg.Market.StorageMarketConfig.MK20.DisabledMiners { + maddr, err := address.NewFromString(m) + if err != nil { + return nil, xerrors.Errorf("failed to parse miner string: %s", err) + } + mk20disabledMiners = append(mk20disabledMiners, maddr) + } + mk20enabled, _ := lo.Difference(miners, mk20disabledMiners) mmap := make(map[string][]address.Address) - mmap[mk12Str] = enabled + mmap[mk12Str] = mk12enabled + mmap[mk20Str] = mk20enabled bmcfg := make(map[address.Address]config.BalanceManagerConfig) for _, a := range cfg.Addresses { if len(a.MinerAddresses) > 0 { diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index f4869baa2..5e17df715 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -176,11 +176,9 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 allocationID = nil } - var aggregation interface{} + aggregation := 0 if data.Format.Aggregate != nil { - aggregation = data.Format.Aggregate.Type - } else { - aggregation = nil + aggregation = int(data.Format.Aggregate.Type) } // Insert pipeline when Data source is HTTP @@ -278,159 +276,200 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 ID string PieceCID cid.Cid Size abi.PaddedPieceSize + RawSize uint64 } toDownload := make(map[downloadkey][]mk20.HttpUrl) - existing := make(map[downloadkey]*int64) - offlinelist := make(map[downloadkey]struct{}) for _, piece := range deal.Data.SourceAggregate.Pieces { if piece.SourceHTTP != nil { - urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] + urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size, RawSize: piece.SourceHTTP.RawSize}] if ok { toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = append(urls, piece.SourceHTTP.URLs...) } else { - toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = piece.SourceHTTP.URLs - existing[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = nil + toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size, RawSize: piece.SourceHTTP.RawSize}] = piece.SourceHTTP.URLs } } - if piece.SourceOffline != nil { - offlinelist[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = struct{}{} - } - } - - pqBatch := &pgx.Batch{} - pqBatchSize := 20000 - - for k, _ := range toDownload { - pqBatch.Queue(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, k.PieceCID.String(), int64(k.Size)).QueryRow(func(row pgx.Row) error { - var id int64 - err = row.Scan(&id) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return nil - } - return xerrors.Errorf("scanning parked piece id: %w", err) - } - existing[k] = &id - return nil - }) - if pqBatch.Len() > pqBatchSize { - res := tx.SendBatch(ctx, pqBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece query batch: %w", err) - } - pqBatch = &pgx.Batch{} - } - } - - if pqBatch.Len() > 0 { - res := tx.SendBatch(ctx, pqBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece query batch: %w", err) - } - } - - piBatch := &pgx.Batch{} - piBatchSize := 10000 - for k, v := range existing { - if v == nil { - piBatch.Queue(`INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - VALUES ($1, $2, $3, FALSE) - ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING - RETURNING id`, k.PieceCID.String(), int64(k.Size), int64(k.Size)).QueryRow(func(row pgx.Row) error { - var id int64 - err = row.Scan(&id) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return nil - } - return xerrors.Errorf("scanning parked piece id: %w", err) - } - v = &id - return nil - }) - if piBatch.Len() > piBatchSize { - res := tx.SendBatch(ctx, piBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece insert batch: %w", err) - } - piBatch = &pgx.Batch{} - } - } - } - - if piBatch.Len() > 0 { - res := tx.SendBatch(ctx, piBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece insert batch: %w", err) - } } - prBatch := &pgx.Batch{} - prBatchSize := 10000 - downloadMap := make(map[downloadkey][]int64) + batch := &pgx.Batch{} + batchSize := 5000 - for k, v := range existing { - if v == nil { - return xerrors.Errorf("missing parked piece for %s", k.PieceCID.String()) - } - var refIds []int64 - urls := toDownload[downloadkey{PieceCID: k.PieceCID, Size: k.Size}] - for _, src := range urls { + for k, v := range toDownload { + for _, src := range v { headers, err := json.Marshal(src.Headers) if err != nil { return xerrors.Errorf("marshal headers: %w", err) } - prBatch.Queue(`INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) VALUES ($1, $2, $3, FALSE) RETURNING ref_id`, - *v, src.URL, headers).QueryRow(func(row pgx.Row) error { - var id int64 - err = row.Scan(&id) - if err != nil { - return xerrors.Errorf("scanning parked piece ref id: %w", err) - } - refIds = append(refIds, id) - return nil - }) - - if prBatch.Len() > prBatchSize { - res := tx.SendBatch(ctx, prBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece ref insert batch: %w", err) - } - prBatch = &pgx.Batch{} - } - } - downloadMap[downloadkey{ID: dealID, PieceCID: k.PieceCID, Size: k.Size}] = refIds - - } - - if prBatch.Len() > 0 { - res := tx.SendBatch(ctx, prBatch) - if err := res.Close(); err != nil { - return xerrors.Errorf("closing parked piece ref insert batch: %w", err) + batch.Queue(`WITH inserted_piece AS ( + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, FALSE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id + ), + selected_piece AS ( + SELECT COALESCE( + (SELECT id FROM inserted_piece), + (SELECT id FROM parked_pieces + WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = FALSE AND cleanup_task_id IS NULL) + ) AS id + ), + inserted_ref AS ( + INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + SELECT id, $4, $5, FALSE FROM selected_piece + RETURNING ref_id + ) + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + VALUES ($6, $1, $2, ARRAY[(SELECT ref_id FROM inserted_ref)]) + ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + SET ref_ids = array_append( + market_mk20_download_pipeline.ref_ids, + (SELECT ref_id FROM inserted_ref) + ) + WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, + k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID) } - } - mdBatch := &pgx.Batch{} - mdBatchSize := 20000 - for k, v := range downloadMap { - mdBatch.Queue(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - k.ID, k.PieceCID.String(), k.Size, v) - if mdBatch.Len() > mdBatchSize { - res := tx.SendBatch(ctx, mdBatch) + if batch.Len() > batchSize { + res := tx.SendBatch(ctx, batch) if err := res.Close(); err != nil { - return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + return xerrors.Errorf("closing parked piece query batch: %w", err) } - mdBatch = &pgx.Batch{} + batch = &pgx.Batch{} } } - if mdBatch.Len() > 0 { - res := tx.SendBatch(ctx, mdBatch) + + if batch.Len() > 0 { + res := tx.SendBatch(ctx, batch) if err := res.Close(); err != nil { - return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + return xerrors.Errorf("closing parked piece query batch: %w", err) } } + //existingCount = 0 + //for _, v := range existing { + // if v != nil { + // existingCount++ + // } + //} + // + //log.Infow("Initial Existing after first pass", "Count", existingCount) + // + //piBatch := &pgx.Batch{} + //piBatchSize := 10000 + //for k, v := range existing { + // if v == nil { + // piBatch.Queue(`WITH inserted_piece AS ( + // INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + // VALUES ($1, $2, $3, FALSE) + // ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + // RETURNING id + // ), + // selected_piece AS ( + // SELECT COALESCE( + // (SELECT id FROM inserted_piece), + // (SELECT id FROM parked_pieces + // WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = FALSE AND cleanup_task_id IS NULL) + // ) AS id + // ), + // inserted_ref AS ( + // INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + // SELECT id, $4, $5, FALSE FROM selected_piece + // RETURNING ref_id + // ) + // INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + // VALUES ($6, $1, $2, ARRAY[(SELECT ref_id FROM inserted_ref)]) + // ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + // SET ref_ids = array_append( + // market_mk20_download_pipeline.ref_ids, + // (SELECT ref_id FROM inserted_ref) + // ) + // WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, + // k.PieceCID.String(), k.Size) + // if piBatch.Len() > piBatchSize { + // res := tx.SendBatch(ctx, piBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing parked piece insert batch: %w", err) + // } + // piBatch = &pgx.Batch{} + // } + // } + //} + // + //if piBatch.Len() > 0 { + // res := tx.SendBatch(ctx, piBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing parked piece insert batch: %w", err) + // } + //} + // + //existingCount = 0 + //for _, v := range existing { + // if v != nil { + // existingCount++ + // } + //} + // + //log.Infow("Initial Existing after second pass", "Count", existingCount) + // + //prBatch := &pgx.Batch{} + //prBatchSize := 10000 + // + //for k, v := range existing { + // if v == nil { + // return xerrors.Errorf("missing parked piece for %s", k.PieceCID.String()) + // } + // urls := toDownload[downloadkey{PieceCID: k.PieceCID, Size: k.Size}] + // for _, src := range urls { + // headers, err := json.Marshal(src.Headers) + // if err != nil { + // return xerrors.Errorf("marshal headers: %w", err) + // } + // prBatch.Queue(`WITH inserted_ref AS ( + // INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + // VALUES ($1, $2, $3, FALSE) + // RETURNING ref_id + // ) + // INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + // VALUES ($4, $5, $6, ARRAY[(SELECT ref_id FROM inserted_ref)])`, + // *v, src.URL, headers, k.ID, k.PieceCID.String(), k.Size) + // } + // + // if prBatch.Len() > 0 { + // res := tx.SendBatch(ctx, prBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing parked piece ref insert batch: %w", err) + // } + // } + //} + // + //if prBatch.Len() > prBatchSize { + // res := tx.SendBatch(ctx, prBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing parked piece ref insert batch: %w", err) + // } + // prBatch = &pgx.Batch{} + //} + + //mdBatch := &pgx.Batch{} + //mdBatchSize := 20000 + //for k, v := range downloadMap { + // mdBatch.Queue(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, + // k.ID, k.PieceCID.String(), k.Size, v) + // if mdBatch.Len() > mdBatchSize { + // res := tx.SendBatch(ctx, mdBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + // } + // mdBatch = &pgx.Batch{} + // } + //} + //if mdBatch.Len() > 0 { + // res := tx.SendBatch(ctx, mdBatch) + // if err := res.Close(); err != nil { + // return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) + // } + //} + pBatch := &pgx.Batch{} pBatchSize := 4000 for i, piece := range deal.Data.SourceAggregate.Pieces { @@ -466,6 +505,8 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 return nil } + // Insert pipeline when data + return xerrors.Errorf("unknown data source type") } @@ -475,7 +516,7 @@ func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { id, sp_id, contract, - piece_index, + client, piece_cid, piece_size, raw_size, @@ -483,12 +524,13 @@ func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { url, indexing, announce, - verified, allocation_id, duration, piece_aggregation, started, downloaded, + commp_task_id, + after_commp, deal_aggregation, aggr_index, agg_task_id, @@ -549,7 +591,7 @@ func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK2 if !piece.Downloaded && piece.Started { _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { var refid int64 - err = tx.QueryRow(`SELECT ref_id FROM ( + err = tx.QueryRow(`SELECT u.ref_id FROM ( SELECT unnest(dp.ref_ids) AS ref_id FROM market_mk20_download_pipeline dp WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 @@ -804,7 +846,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie func (d *CurioStorageDealMarket) createCommPMk20Piece(ctx context.Context, piece MK20PipelinePiece) error { if piece.Downloaded && !piece.AfterCommp && piece.CommTaskID == nil { // Skip commP is configured to do so - if d.cfg.Market.StorageMarketConfig.MK12.SkipCommP { + if d.cfg.Market.StorageMarketConfig.MK20.SkipCommP { _, err := d.db.Exec(ctx, `UPDATE market_mk20_pipeline SET after_commp = TRUE, commp_task_id = NULL WHERE id = $1 AND sp_id = $2 @@ -817,7 +859,7 @@ func (d *CurioStorageDealMarket) createCommPMk20Piece(ctx context.Context, piece if err != nil { return xerrors.Errorf("marking piece as after commP: %w", err) } - log.Infow("commP skipped successfully", "deal piece", piece) + log.Debugw("commP skipped successfully", "deal piece", piece) return nil } @@ -833,15 +875,18 @@ func (d *CurioStorageDealMarket) createCommPMk20Piece(ctx context.Context, piece AND aggr_index = $7 AND downloaded = TRUE AND after_commp = FALSE - AND commp_task_id = NULL`, id, piece.ID, piece.SPID, piece.PieceCID, piece.PieceSize, piece.RawSize, piece.AggregationIndex) + AND commp_task_id IS NULL`, id, piece.ID, piece.SPID, piece.PieceCID, piece.PieceSize, piece.RawSize, piece.AggregationIndex) if err != nil { return false, xerrors.Errorf("creating commP task for deal piece: %w", err) } + if n > 0 { + log.Debugw("commP task created successfully", "deal piece", piece) + } + // commit only if we updated the piece return n > 0, nil }) - log.Infow("commP task created successfully", "deal piece", piece) } return nil @@ -851,7 +896,7 @@ func (d *CurioStorageDealMarket) createCommPMk20Piece(ctx context.Context, piece func (d *CurioStorageDealMarket) addDealOffset(ctx context.Context, piece MK20PipelinePiece) error { // Get the deal offset if sector has started sealing - if piece.Sector != nil && piece.RegSealProof == nil { + if piece.Sector != nil && piece.RegSealProof != nil && piece.SectorOffset == nil { _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { type pieces struct { Cid string `db:"piece_cid"` @@ -929,15 +974,21 @@ func (d *CurioStorageDealMarket) processMK20DealAggregation(ctx context.Context) } for _, deal := range deals { + log.Infow("processing aggregation task", "deal", deal.ID, "count", deal.Count) d.adders[pollerAggregate].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, err error) { n, err := tx.Exec(`UPDATE market_mk20_pipeline SET agg_task_id = $1 WHERE id = $2 AND after_commp = TRUE - AND NOT aggregated + AND aggregated = FALSE AND agg_task_id IS NULL`, id, deal.ID) if err != nil { return false, xerrors.Errorf("creating aggregation task for deal: %w", err) } + + if n == deal.Count { + log.Infow("aggregation task created successfully", "deal", deal.ID) + } + return n == deal.Count, nil }) } @@ -1002,7 +1053,12 @@ func (d *CurioStorageDealMarket) processMK20DealIngestion(ctx context.Context) { continue } - clientId, err := address.IDFromAddress(client) + clientIdAddr, err := d.api.StateLookupID(ctx, client, types.EmptyTSK) + if err != nil { + log.Errorw("failed to lookup client id", "deal", deal, "error", err) + } + + clientId, err := address.IDFromAddress(clientIdAddr) if err != nil { log.Errorw("failed to parse client id", "deal", deal, "error", err) continue @@ -1066,14 +1122,16 @@ func (d *CurioStorageDealMarket) processMK20DealIngestion(ctx context.Context) { if err != nil { return false, xerrors.Errorf("failed to allocate piece to sector: %w", err) } - n, err := tx.Exec(`UPDATE market_mk20_pipeline SET SET sector = $1, reg_seal_proof = $2 WHERE id = $3`, *sector, *sp, deal.ID) + + n, err := tx.Exec(`UPDATE market_mk20_pipeline SET sector = $1, reg_seal_proof = $2 WHERE id = $3`, *sector, *sp, deal.ID) if err != nil { return false, xerrors.Errorf("failed to update deal: %w", err) } + return n == 1, nil }, harmonydb.OptionRetry()) if err != nil { - log.Errorf("failed to commit transaction: %w", err) + log.Errorf("failed to commit transaction: %s", err) continue } if comm { diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index 373417f43..1246c9236 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -58,7 +58,7 @@ const ( numPollers ) -const dealPollerInterval = 30 * time.Second +const dealPollerInterval = 3 * time.Second type storageMarketAPI interface { mk12.MK12API @@ -118,7 +118,8 @@ type MK12Pipeline struct { func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector, stor paths.StashStore) *CurioStorageDealMarket { moduleMap := make(map[string][]address.Address) - moduleMap[mk12Str] = append(moduleMap[mk12Str], miners...) + moduleMap[mk12Str] = miners + moduleMap[mk20Str] = miners urls := make(map[string]http.Header) for _, curl := range cfg.Market.StorageMarketConfig.PieceLocator { diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index f6df5810b..1f4b6d36d 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -35,17 +35,15 @@ type AggregateTask struct { sc *ffi.SealCalls stor paths.StashStore api headAPI - max int } -func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.SealCalls, stor paths.StashStore, api headAPI, max int) *AggregateTask { +func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.SealCalls, stor paths.StashStore, api headAPI) *AggregateTask { return &AggregateTask{ sm: sm, db: db, sc: sc, stor: stor, api: api, - max: max, } } @@ -53,27 +51,28 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d ctx := context.Background() var pieces []struct { - Pcid string `db:"piece_cid"` - Psize int64 `db:"piece_size"` - RawSize int64 `db:"raw_size"` - URL string `db:"url"` - ID string `db:"id"` - SpID int64 `db:"sp_id"` - AggrIndex int `db:"aggr_index"` - Aggregated bool `db:"aggregated"` - Aggreation int `db:"deal_aggregation"` + Pcid string `db:"piece_cid"` + Psize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + URL string `db:"url"` + ID string `db:"id"` + SpID int64 `db:"sp_id"` + AggrIndex int `db:"aggr_index"` + Aggregated bool `db:"aggregated"` + Aggregation int `db:"deal_aggregation"` } err = a.db.Select(ctx, &pieces, ` SELECT + piece_cid, + piece_size, + raw_size, url, - headers, - raw_size, - piece_cid, - piece_size, id, sp_id, - aggr_index + aggr_index, + aggregated, + deal_aggregation FROM market_mk20_pipeline WHERE @@ -121,11 +120,13 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d var pinfos []abi.PieceInfo var readers []io.Reader + var refIDs []int64 + for _, piece := range pieces { if piece.Aggregated { return false, xerrors.Errorf("piece %s for deal %s already aggregated for task %d", piece.Pcid, piece.ID, taskID) } - if piece.Aggreation != 1 { + if piece.Aggregation != 1 { return false, xerrors.Errorf("incorrect aggregation value for piece %s for deal %s for task %d", piece.Pcid, piece.ID, taskID) } if piece.ID != id || piece.SpID != spid { @@ -183,6 +184,7 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d pReader, _ := padreader.New(reader, uint64(piece.RawSize)) readers = append(readers, pReader) + refIDs = append(refIDs, refNum) } _, aggregatedRawSize, err := datasegment.ComputeDealPlacement(pinfos) @@ -283,6 +285,11 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d return false, fmt.Errorf("failed to delete pipeline pieces: %w", err) } + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1) AND long_term = FALSE`, refIDs) + if err != nil { + return false, fmt.Errorf("failed to delete parked_piece_refs entries: %w", err) + } + ddo := deal.Products.DDOV1 data := deal.Data @@ -298,7 +305,7 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, TRUE, TRUE, TRUE, TRUE)`, - id, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(data.SourceHTTP.RawSize), pieceIDUrl.String(), + id, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, rawSize, pieceIDUrl.String(), false, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, data.Format.Aggregate.Type, data.Format.Aggregate.Type) if err != nil { @@ -326,7 +333,7 @@ func (a *AggregateTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask. func (a *AggregateTask) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ - Max: taskhelp.Max(a.max), + Max: taskhelp.Max(50), Name: "AggregateDeals", Cost: resources.Resources{ Cpu: 1, diff --git a/tasks/storage-market/task_commp.go b/tasks/storage-market/task_commp.go index fb828dec3..11758f285 100644 --- a/tasks/storage-market/task_commp.go +++ b/tasks/storage-market/task_commp.go @@ -3,6 +3,7 @@ package storage_market import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -10,6 +11,7 @@ import ( "strconv" "github.com/ipfs/go-cid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-commp-utils/writer" @@ -50,32 +52,26 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done ctx := context.Background() var pieces []struct { - Pcid string `db:"piece_cid"` - Psize int64 `db:"piece_size"` - RawSize int64 `db:"raw_size"` - URL *string `db:"url"` - Headers json.RawMessage `db:"headers"` - UUID *string `db:"uuid"` // Nullable because it only exists in market_mk12_deal_pipeline - ID *string `db:"id"` // Nullable because it only exists in market_mk20_pipeline - IDType *int `db:"id_type"` - SpID *int64 `db:"sp_id"` - Contract *string `db:"contract"` - PieceIndex *int `db:"piece_index"` - MK12Piece bool `db:"mk12_source_table"` + Pcid string `db:"piece_cid"` + Psize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + URL *string `db:"url"` + Headers json.RawMessage `db:"headers"` + ID string `db:"id"` + SpID int64 `db:"sp_id"` + MK12Piece bool `db:"mk12_source_table"` + AggrIndex int64 `db:"aggr_index"` } err = c.db.Select(ctx, &pieces, `SELECT - uuid, + uuid AS id, url, headers, raw_size, piece_cid, piece_size, - NULL AS id, - NULL AS id_type, - NULL AS sp_id, - NULL AS contract, - NULL AS piece_index, + sp_id, + 0 AS aggr_index, TRUE AS mk12_source_table FROM market_mk12_deal_pipeline @@ -85,17 +81,14 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done UNION ALL SELECT - NULL AS uuid, + id, url, - headers, + NULL AS headers, raw_size, piece_cid, - piece_size, - id, - id_type, - sp_id, - contract, - piece_index, + piece_size, + sp_id, + aggr_index, FALSE AS mk12_source_table FROM market_mk20_pipeline @@ -110,20 +103,13 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done piece := pieces[0] if piece.MK12Piece { - if piece.UUID == nil { - return false, xerrors.Errorf("expected UUID to be non-null for mk12 piece") - } - expired, err := checkExpiry(ctx, c.db, c.api, *piece.UUID, c.sm.pin.GetExpectedSealDuration()) + expired, err := checkExpiry(ctx, c.db, c.api, piece.ID, c.sm.pin.GetExpectedSealDuration()) if err != nil { - return false, xerrors.Errorf("deal %s expired: %w", *piece.UUID, err) + return false, xerrors.Errorf("deal %s expired: %w", piece.ID, err) } if expired { return true, nil } - } else { - if piece.ID == nil || piece.IDType == nil || piece.SpID == nil || piece.Contract == nil || piece.PieceIndex == nil { - return false, xerrors.Errorf("expected ID, IDType, SpID, Contract, PieceIndex to be non-null for mk20 piece") - } } if piece.URL != nil { @@ -249,18 +235,17 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done if piece.MK12Piece { n, err = c.db.Exec(ctx, `UPDATE market_mk12_deal_pipeline SET after_commp = TRUE, psd_wait_time = NOW(), commp_task_id = NULL WHERE commp_task_id = $1`, taskID) } else { - n, err = c.db.Exec(ctx, `UPDATE market_mk20_pipeline SET after_commp = TRUE, commp_task_id = $9 + n, err = c.db.Exec(ctx, `UPDATE market_mk20_pipeline SET after_commp = TRUE, commp_task_id = NULL WHERE id = $1 - AND id_type = $2 - AND sp_id = $3 - AND contract = $4 - AND piece_cid = $5 - AND piece_size = $6 - AND raw_size = $7 - AND piece_index = $8 + AND sp_id = $2 + AND piece_cid = $3 + AND piece_size = $4 + AND raw_size = $5 + AND aggr_index = $6 AND downloaded = TRUE - AND after_commp = FALSE`, - *piece.ID, *piece.IDType, *piece.SpID, *piece.Contract, piece.Pcid, piece.Psize, piece.RawSize, *piece.PieceIndex, taskID) + AND after_commp = FALSE + AND commp_task_id = $7`, + piece.ID, piece.SpID, piece.Pcid, piece.Psize, piece.RawSize, piece.AggrIndex, taskID) } if err != nil { @@ -277,7 +262,7 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("failed to find URL for the piece %s in the db", piece.Pcid) } - return false, xerrors.Errorf("failed to find URL for the mk20 deal piece with id %s, idType %d, SP %d, Contract %s, Index %d and CID %s in the db", *piece.ID, *piece.IDType, *piece.SpID, *piece.Contract, *piece.PieceIndex, piece.Pcid) + return false, xerrors.Errorf("failed to find URL for the mk20 deal piece with id %s, SP %d, CID %s, Size %d and Index %d in the db", piece.ID, piece.SpID, piece.Pcid, piece.Psize, piece.AggrIndex) } @@ -292,7 +277,6 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task var tasks []struct { TaskID harmonytask.TaskID `db:"commp_task_id"` - SpID int64 `db:"sp_id"` StorageID string `db:"storage_id"` Url *string `db:"url"` } @@ -303,9 +287,8 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task } comm, err := c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - err = tx.Select(&tasks, `SELECT + err = tx.Select(&tasks, ` SELECT commp_task_id, - sp_id, url FROM market_mk12_deal_pipeline @@ -316,7 +299,6 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task SELECT commp_task_id, - sp_id, url FROM market_mk20_pipeline @@ -356,7 +338,7 @@ func (c *CommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task err = tx.QueryRow(` SELECT storage_id FROM sector_location - WHERE miner_id = $1 AND sector_num = $2 AND l.sector_filetype = 32`, task.SpID, pieceID[0].PieceID).Scan(&sLocation) + WHERE miner_id = 0 AND sector_num = $1 AND sector_filetype = 32`, pieceID[0].PieceID).Scan(&sLocation) if err != nil { return false, xerrors.Errorf("failed to get storage location from DB: %w", err) @@ -460,11 +442,11 @@ func checkExpiry(ctx context.Context, db *harmonydb.DB, api headAPI, deal string var starts []struct { StartEpoch int64 `db:"start_epoch"` } - err := db.Select(ctx, &starts, `SELECT start_epoch FROM market_mk12_deals WHERE uuid = $1 - UNION ALL - SELECT start_epoch FROM market_direct_deals WHERE uuid = $1 - LIMIT 1`, deal) + err := db.Select(ctx, &starts, `SELECT start_epoch FROM market_mk12_deals WHERE uuid = $1 LIMIT 1`, deal) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } return false, xerrors.Errorf("failed to get start epoch from DB: %w", err) } if len(starts) != 1 { diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index fe9c8a771..757d21b84 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -132,7 +132,7 @@ func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { CIDCount int64 `db:"cid_count"` } - err = a.deps.DB.Select(ctx, &adEntryInfo, `SELECT count(1) as entry_count, sum(num_blocks) as cid_count from ipni_chunks where piece_cid=$1`, details.PieceCid) + err = a.deps.DB.Select(ctx, &adEntryInfo, `SELECT count(1) as entry_count, sum(num_blocks) as cid_count from ipni_chunks where piece_cid=$1`, details.PieceCidV2) if err != nil { return nil, xerrors.Errorf("failed to fetch the ad entry count from DB: %w", err) } diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index b1e019c97..59f47cfa3 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "net/http" "strconv" @@ -624,13 +625,14 @@ type PieceDeal struct { } type PieceInfo struct { - PieceCid string `json:"piece_cid"` - Size int64 `json:"size"` - CreatedAt time.Time `json:"created_at"` - Indexed bool `json:"indexed"` - IndexedAT time.Time `json:"indexed_at"` - IPNIAd string `json:"ipni_ad"` - Deals []*PieceDeal `json:"deals"` + PieceCidv2 string `json:"piece_cid_v2"` + PieceCid string `json:"piece_cid"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` + Indexed bool `json:"indexed"` + IndexedAT time.Time `json:"indexed_at"` + IPNIAd string `json:"ipni_ad"` + Deals []*PieceDeal `json:"deals"` } func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, error) { @@ -646,10 +648,12 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er pi := commp.PieceInfo() - ret := &PieceInfo{} + ret := &PieceInfo{ + PieceCidv2: piece.String(), + } err = a.deps.DB.QueryRow(ctx, `SELECT created_at, indexed, indexed_at FROM market_piece_metadata WHERE piece_cid = $1 AND piece_size = $2`, pi.PieceCID.String(), pi.Size).Scan(&ret.CreatedAt, &ret.Indexed, &ret.IndexedAT) - if err != nil && err != pgx.ErrNoRows { + if err != nil && !errors.Is(err, pgx.ErrNoRows) { return nil, xerrors.Errorf("failed to get piece metadata: %w", err) } @@ -680,7 +684,7 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er ret.Size = pieceDeals[i].Length } ret.Deals = pieceDeals - ret.PieceCid = piece.String() + ret.PieceCid = pi.PieceCID.String() b := new(bytes.Buffer) @@ -893,15 +897,23 @@ type MK20DealPipeline struct { CreatedAt time.Time `db:"created_at" json:"created_at"` } +type PieceInfoMK12Deals struct { + Deal *MK12Deal `json:"deal"` + Pipeline *MK12DealPipeline `json:"mk12_pipeline,omitempty"` +} + +type PieceInfoMK20Deals struct { + Deal *MK20StorageDeal `json:"deal"` + Pipeline *MK20DealPipeline `json:"mk20_pipeline,omitempty"` +} + // PieceDealDetailEntry combines a deal and its pipeline type PieceDealDetailEntry struct { - MK12Deal *MK12Deal `json:"mk12_deal"` - MK12Pipeline *MK12DealPipeline `json:"mk12_pipeline,omitempty"` - MK20Deal *mk20.Deal `json:"mk20_deal,omitempty"` - MK20DealPipeline *MK20DealPipeline `json:"mk20_pipeline,omitempty"` + MK12 []PieceInfoMK12Deals `json:"mk12"` + MK20 []PieceInfoMK20Deals `json:"mk20"` } -func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceDealDetailEntry, error) { +func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDealDetailEntry, error) { pcid, err := cid.Parse(pieceCid) if err != nil { return nil, err @@ -1029,27 +1041,40 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceD pipelineMap[pipeline.UUID] = pipeline } - var entries []PieceDealDetailEntry + ret := &PieceDealDetailEntry{ + MK12: make([]PieceInfoMK12Deals, len(mk12Deals)), + } + for _, deal := range mk12Deals { - entry := PieceDealDetailEntry{ - MK12Deal: deal, + entry := PieceInfoMK12Deals{ + Deal: deal, } if pipeline, exists := pipelineMap[deal.UUID]; exists { - entry.MK12Pipeline = &pipeline + entry.Pipeline = &pipeline } else { - entry.MK12Pipeline = nil // Pipeline may not exist for processed and active deals + entry.Pipeline = nil // Pipeline may not exist for processed and active deals } - entries = append(entries, entry) + ret.MK12 = append(ret.MK12, entry) } var mk20Deals []*mk20.DBDeal - err = a.deps.DB.Select(ctx, &mk20Deals, `SELECT * FROM market_mk20_deals WHERE piece_cid = $1 AND piece_size = $2`, pieceCid) + err = a.deps.DB.Select(ctx, &mk20Deals, `SELECT + id, + piece_cid, + piece_size, + format, + source_http, + source_aggregate, + source_offline, + source_http_put, + ddo_v1, + error FROM market_mk20_deal WHERE piece_cid = $1 AND piece_size = $2`, pieceCid, size) if err != nil { return nil, xerrors.Errorf("failed to query mk20 deals: %w", err) } ids := make([]string, len(mk20Deals)) - mk20deals := make([]*mk20.Deal, len(mk20Deals)) + mk20deals := make([]*MK20StorageDeal, len(mk20Deals)) for i, dbdeal := range mk20Deals { deal, err := dbdeal.ToDeal() @@ -1057,10 +1082,13 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceD return nil, err } ids[i] = deal.Identifier.String() - mk20deals[i] = deal + mk20deals[i] = &MK20StorageDeal{ + Deal: deal, + Error: dbdeal.Error, + } } - var mk20Pipelines []*MK12DealPipeline + var mk20Pipelines []*MK20DealPipeline err = a.deps.DB.Select(ctx, &mk20Pipelines, ` SELECT created_at, @@ -1094,7 +1122,7 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceD indexed, complete FROM market_mk20_pipeline - WHERE id = ANY($1)`) + WHERE id = ANY($1)`, ids) if err != nil { return nil, xerrors.Errorf("failed to query mk20 pipelines: %w", err) } @@ -1106,18 +1134,21 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) ([]PieceD } for _, deal := range mk20deals { - entry := PieceDealDetailEntry{ - MK20Deal: deal, + entry := PieceInfoMK20Deals{ + Deal: deal, } - if pipeline, exists := mk20pipelineMap[deal.Identifier.String()]; exists { - entry.MK20DealPipeline = &pipeline + if pipeline, exists := mk20pipelineMap[deal.Deal.Identifier.String()]; exists { + entry.Pipeline = &pipeline } else { - entry.MK20DealPipeline = nil // Pipeline may not exist for processed and active deals + entry.Pipeline = nil // Pipeline may not exist for processed and active deals } - entries = append(entries, entry) + if ret.MK20 == nil { + ret.MK20 = make([]PieceInfoMK20Deals, 0) + } + ret.MK20 = append(ret.MK20, entry) } - return entries, nil + return ret, nil } func firstOrZero[T any](a []T) T { diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index 7f93ce22b..8b64ce88b 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -17,18 +17,28 @@ import ( ) type MK20StorageDeal struct { - Deal *mk20.Deal `json:"deal"` - Error sql.NullString `json:"error"` + Deal *mk20.Deal `json:"deal"` + Error sql.NullString `json:"error"` + PieceCidV2 string `json:"piece_cid_v2"` } -func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, idStr string) (*MK20StorageDeal, error) { - id, err := ulid.Parse(idStr) +func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20StorageDeal, error) { + pid, err := ulid.Parse(id) if err != nil { return nil, xerrors.Errorf("parsing deal ID: %w", err) } var dbDeal []mk20.DBDeal - err = a.deps.DB.Select(ctx, &dbDeal, `SELECT * FROM market_mk20_deal WHERE id = $1`, id.String()) + err = a.deps.DB.Select(ctx, &dbDeal, `SELECT id, + piece_cid, + piece_size, + format, + source_http, + source_aggregate, + source_offline, + source_http_put, + ddo_v1, + error FROM market_mk20_deal WHERE id = $1`, pid.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -40,30 +50,45 @@ func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, idStr string) (*MK20Sto return nil, xerrors.Errorf("converting DB deal to struct: %w", err) } - return &MK20StorageDeal{Deal: deal, Error: dbDeal[0].Error}, nil + pi := abi.PieceInfo{ + PieceCID: deal.Data.PieceCID, + Size: deal.Data.Size, + } + + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return nil, xerrors.Errorf("failed to get commp: %w", err) + } + + return &MK20StorageDeal{Deal: deal, Error: dbDeal[0].Error, PieceCidV2: commp.PCidV2().String()}, nil } func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) ([]*StorageDealList, error) { var mk20Summaries []*StorageDealList err := a.deps.DB.Select(ctx, &mk20Summaries, `SELECT - d.id as uuid, - d.piece_cid, - d.size AS piece_size, - d.created_at, - d.sp_id, - d.error, - CASE - WHEN w.id IS NOT NULL THEN FALSE - WHEN p.id IS NOT NULL THEN p.complete - ELSE TRUE - END AS processed - FROM market_mk20_deal d - LEFT JOIN market_mk20_pipeline_waiting w ON d.id = w.id - LEFT JOIN market_mk20_pipeline p ON d.id = p.id - WHERE d.ddo_v1 IS NOT NULL AND d.ddo_v1 != 'null' - ORDER BY d.created_at DESC - LIMIT $1 OFFSET $2;`, limit, offset) + d.id AS uuid, + d.piece_cid, + d.piece_size, + d.created_at, + d.sp_id, + d.error, + CASE + WHEN EXISTS ( + SELECT 1 FROM market_mk20_pipeline_waiting w + WHERE w.id = d.id + ) THEN FALSE + WHEN EXISTS ( + SELECT 1 FROM market_mk20_pipeline p + WHERE p.id = d.id AND p.complete = FALSE + ) THEN FALSE + ELSE TRUE + END AS processed + FROM market_mk20_deal d + WHERE d.ddo_v1 IS NOT NULL AND d.ddo_v1 != 'null' + ORDER BY d.created_at DESC + LIMIT $1 OFFSET $2; + `, limit, offset) if err != nil { return nil, fmt.Errorf("failed to fetch deal list: %w", err) } diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index f08232b0a..6d35c9048 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -8,43 +8,51 @@ import '/ux/yesno.mjs'; class DealDetails extends LitElement { constructor() { super(); - this.loadData(); + this.loaddata(); } - async loadData() { + async loaddata() { try { const params = new URLSearchParams(window.location.search); this.data = await RPCCall('MK20DDOStorageDeal', [params.get('id')]); - setTimeout(() => this.loadData(), 10000); this.requestUpdate(); } catch (error) { - alert('Failed to load deal details: ' + error); console.error('Failed to load deal details:', error); + alert(`Failed to load deal details: ${error.message}`); } } render() { + console.log(this.data); if (!this.data) return html`

    No data.

    `; - const { Identifier, Data, Products } = this.data.deal; + const { identifier, data, products } = this.data.deal; + return html` + + +
    Deal
    - - +
    Identifier${Identifier}
    + - - + +
    Identifier${identifier}
    Error
    PieceCID${Data?.piece_cid['/']}
    Size${Data?.size}
    PieceCID${data?.piece_cid['/']}
    PieceSize${data?.piece_size}
    - ${this.renderPieceFormat(Data?.format)} - ${Data?.source_http ? this.renderSourceHTTP(Data.source_http) : ''} - ${Data?.source_aggregate ? this.renderSourceAggregate(Data.source_aggregate) : ''} - ${Data?.source_offline ? this.renderSourceOffline(Data.source_offline) : ''} - ${Data?.source_httpput ? this.renderSourceHttpPut(Data.source_httpput) : ''} + ${this.renderPieceFormat(data?.format)} + ${data?.source_http ? this.renderSourceHTTP(data.source_http) : ''} + ${data?.source_aggregate ? this.renderSourceAggregate(data.source_aggregate) : ''} + ${data?.source_offline ? this.renderSourceOffline(data.source_offline) : ''} + ${data?.source_httpput ? this.renderSourceHttpPut(data.source_httpput) : ''} - ${Products?.ddo_v1 ? this.renderDDOV1(Products.ddo_v1) : ''} + ${products?.ddo_v1 ? this.renderDDOV1(products.ddo_v1) : ''}
    `; } @@ -53,7 +61,7 @@ class DealDetails extends LitElement { if (!format) return ''; return html`
    Piece Format
    - +
    ${format.car ? html`` : ''} ${format.aggregate ? html` @@ -70,7 +78,7 @@ class DealDetails extends LitElement { if (!subs?.length) return ''; return html`
    Aggregate Sub Formats
    -
    CarYes
    +
    ${subs.map((s, i) => html` @@ -89,12 +97,11 @@ class DealDetails extends LitElement { renderSourceHTTP(src) { return html`
    Source HTTP
    -
    #CarRawAggregate
    +
    - - + + + + ` + } + if (data.source_aggregate) { + return html` + + + + + ` + } + if (data.source_offline) { + return html` + + + + + ` + } + if (data.source_httpput) { + return html` + + + + + ` + } + } + renderPieceFormat(format) { if (!format) return ''; return html` -
    Piece Format
    Raw Size${src.rawsize}
    - URLs - + + - + @@ -133,6 +133,16 @@ class MK12DealList extends LitElement { } } + formatPieceCid(pieceCid) { + if (!pieceCid) return ''; + if (pieceCid.length <= 24) { + return pieceCid; + } + const start = pieceCid.substring(0, 16); + const end = pieceCid.substring(pieceCid.length - 8); + return `${start}...${end}`; + } + static styles = css` .pagination-controls { display: flex; diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 6d35c9048..24f14045b 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -26,7 +26,7 @@ class DealDetails extends LitElement { console.log(this.data); if (!this.data) return html`

    No data.

    `; - const { identifier, data, products } = this.data.deal; + const { identifier, data, products, error } = this.data.deal; return html` @@ -37,39 +37,85 @@ class DealDetails extends LitElement { /> -
    -
    Deal
    URLs + ${src.urls.map(u => html` @@ -116,9 +123,9 @@ class DealDetails extends LitElement { return html`
    Source Aggregate
    ${src.pieces.map((piece, i) => html` -
    +
    Piece ${i + 1} -
    URLPriorityFallback
    +
    PieceCID${piece.piece_cid['/']}
    Size${piece.size}
    @@ -130,7 +137,7 @@ class DealDetails extends LitElement { renderSourceOffline(src) { return html`
    Source Offline
    - +
    Raw Size${src.raw_size}
    `; @@ -139,7 +146,7 @@ class DealDetails extends LitElement { renderSourceHttpPut(src) { return html`
    Source HTTP PUT
    - +
    Raw Size${src.raw_size}
    `; @@ -148,7 +155,7 @@ class DealDetails extends LitElement { renderDDOV1(ddo) { return html`
    DDO v1
    - +
    @@ -238,9 +245,9 @@ customElements.define('deal-details', DealDetails); // // ${this.deal.data ? html` // -// +// // -// ${this.renderNested('Data', this.deal.data)} +// ${this.renderNested('data', this.deal.data)} // ` : null} // ${this.deal.products?.ddo_v1 ? html` // diff --git a/web/static/pages/mk20-deal/index.html b/web/static/pages/mk20-deal/index.html index f234b04ee..b29fb114d 100644 --- a/web/static/pages/mk20-deal/index.html +++ b/web/static/pages/mk20-deal/index.html @@ -27,7 +27,6 @@

    MK20 Deal Info

    - diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index 27489d6ca..313ad543d 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -30,7 +30,7 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { // Fetch piece info this.data = await RPCCall('PieceInfo', [pieceCid]); - this.mk12DealData = await RPCCall('MK12DealDetail', [pieceCid]); + this.DealData = await RPCCall('PieceDealDetail', [pieceCid]); this.pieceParkStates = await RPCCall('PieceParkStates', [pieceCid]); // TODO SNAP/POREP pipelines @@ -79,6 +79,10 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement {
    Provider${ddo.provider}
    Client${ddo.client}
    Piece Manager${ddo.piece_manager}
    Identifier${this.deal.identifier}
    Datadata
    + + + + @@ -137,9 +141,9 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { ${this.pieceParkStates ? this.renderPieceParkStates() : ''} - ${this.mk12DealData && this.mk12DealData.length > 0 ? html` -

    Related Deals

    - ${this.mk12DealData.map((entry) => html` + ${this.DealData && this.DealData.mk12.length > 0 ? html` +

    Related MK12 Deals

    + ${this.DealData.mk12.map((entry) => html`

    Deal ${entry.deal.uuid}

    Piece CID${this.data.piece_cid_v2}
    Piece CID V1 ${this.data.piece_cid}
    @@ -319,6 +323,170 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement {
    Top Level Info šŸ“‹
    `)} ` : ''} + + ${this.DealData && this.DealData.mk20.length > 0 ? html` +

    Related MK20 Deals

    + ${this.DealData.mk20.map((entry) => html` +

    Deal ${entry.deal.deal.identifier}

    + + + + + + + + + + + + + + + + + ${(() => { + const matchingPieceDeals = this.data.deals.filter(deal => deal.id === entry.deal.uuid); + if (matchingPieceDeals.length > 0) { + return html` + + + `; + } + })()} + ${entry.pipeline ? html` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ` : html` + + `} + +
    Top Level Info šŸ“‹
    ID${entry.deal.deal.identifier}
    Deal Data āš™ļø
    Piece CID${entry.deal.deal.data.piece_cid}
    Piece Size${this.toHumanBytes(entry.deal.deal.data.piece_size)}
    Data Source šŸ“„ļø
    URL Headers +
    + [SHOW] + 
    ${JSON.stringify(entry.deal.url_headers, null, 2)}
    +
    +
    Status šŸŸ¢ļøšŸ”“
    Error${entry.deal.error.Valid ? entry.deal.error.String : 'N/A'}
    Associated Piece Deals šŸ”—ļø
    + + + + + + + + + + + + + + + ${matchingPieceDeals.map((item) => html` + + + + + + + + + + + `)} + +
    IDDeal TypeMinerChain Deal IDSectorOffsetLengthRaw Size
    ${item.id}${item.boost_deal ? 'Boost' : (item.legacy_deal ? 'Legacy' : 'DDO')}${item.miner}${item.chain_deal_id}${item.sector}${item.offset}${this.toHumanBytes(item.length)}${this.toHumanBytes(item.raw_size)}
    +
    PIPELINE ACTIVE
    Controls + +
    Created At${formatDate(entry.pipeline.created_at)}
    Piece CID${entry.pipeline.piece_cid}
    Piece Size${this.toHumanBytes(entry.pipeline.piece_size)}
    Raw Size${entry.pipeline.raw_size.Valid ? this.toHumanBytes(entry.pipeline.raw_size.Int64) : 'N/A'}
    Offline
    URL${entry.pipeline.url.Valid ? entry.pipeline.url.String : 'N/A'}
    Headers
    ${JSON.stringify(entry.pipeline.headers, null, 2)}
    Should Index${this.renderNullableYesNo(entry.pipeline.should_index.Bool)}
    Announce${this.renderNullableYesNo(entry.pipeline.announce.Bool)}
    Progress šŸ› ļø
    Data Fetched${this.renderNullableDoneNotDone(entry.pipeline.started.Bool)}
    After Commp${this.renderNullableDoneNotDone(entry.pipeline.after_commp.Bool)}
    After PSD${this.renderNullableDoneNotDone(entry.pipeline.after_psd.Bool)}
    After Find Deal${this.renderNullableDoneNotDone(entry.pipeline.after_find_deal.Bool)}
    Sealed${this.renderNullableDoneNotDone(entry.pipeline.sealed.Bool)}
    Indexed${this.renderNullableDoneNotDone(entry.pipeline.indexed.Bool)}
    Announced
    Early States šŸŒæ
    Commp Task ID + ${entry.pipeline.commp_task_id.Valid + ? html`` + : 'N/A'} +
    PSD Task ID + ${entry.pipeline.psd_task_id.Valid + ? html`` + : 'N/A'} +
    PSD Wait Time${entry.pipeline.psd_wait_time.Valid ? formatDate(entry.pipeline.psd_wait_time.Time) : 'N/A'}
    Find Deal Task ID + ${entry.pipeline.find_deal_task_id.Valid + ? html`` + : 'N/A'} +
    Sealing šŸ“¦
    Sector${entry.pipeline.sector.Valid ? html`${entry.pipeline.sector.Int64}` : 'N/A'}
    Reg Seal Proof${entry.pipeline.reg_seal_proof.Valid ? entry.pipeline.reg_seal_proof.Int64 : 'N/A'}
    Sector Offset${entry.pipeline.sector_offset.Valid ? entry.pipeline.sector_offset.Int64 : 'N/A'}
    Indexing šŸ”
    Indexing Created At${entry.pipeline.indexing_created_at.Valid ? formatDate(entry.pipeline.indexing_created_at.Time) : 'N/A'}
    Indexing Task ID + ${entry.pipeline.indexing_task_id.Valid + ? html`` + : 'N/A'} +
    No Pipeline Data
    + `)} + ` : ''} `; } From f3e99f63244b1d2e6c026b80b43f4e7800f19d66 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 27 May 2025 16:41:10 +0400 Subject: [PATCH 08/55] offline deal and put deals --- cmd/sptool/toolbox_deal_client.go | 71 ++++-- deps/deps.go | 5 +- .../piece-server/sample/mk20-random-deal.sh | 35 ++- documentation/en/curio-cli/sptool.md | 28 ++- .../harmonydb/sql/20250505-market_mk20.sql | 69 ++++++ itests/curio_test.go | 3 +- lib/cachedreader/cachedreader.go | 102 +++++--- lib/testutils/testutils.go | 2 +- .../{create.cql => cql/0001_create.cql} | 0 market/indexstore/cql/0002_piece_index.cql | 7 + market/indexstore/indexstore.go | 129 ++++++++-- market/indexstore/indexstore_test.go | 9 +- market/ipni/chunker/serve-chunker.go | 2 +- market/mk20/http/http.go | 11 +- market/mk20/http/info.md | 2 +- market/mk20/mk20.go | 9 + market/mk20/mk20_utils.go | 92 +++++-- market/mk20/utils.go | 8 + market/retrieval/piecehandler.go | 13 +- .../remoteblockstore/remoteblockstore.go | 9 +- tasks/gc/pipeline_meta_gc.go | 17 +- tasks/indexing/task_indexing.go | 67 ++++-- tasks/indexing/task_ipni.go | 14 +- tasks/pdp/task_prove.go | 8 +- tasks/storage-market/mk20.go | 225 ++---------------- 25 files changed, 568 insertions(+), 369 deletions(-) rename market/indexstore/{create.cql => cql/0001_create.cql} (100%) create mode 100644 market/indexstore/cql/0002_piece_index.cql diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 7fdd69c2d..af0d82e5c 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -1650,6 +1650,11 @@ var mk20DealCmd = &cli.Command{ Name: "aggregate", Usage: "aggregate file path for the deal", }, + &cli.BoolFlag{ + Name: "put", + Usage: "used HTTP put as data source", + Value: false, + }, }, Action: func(cctx *cli.Context) error { ctx := cctx.Context @@ -1733,11 +1738,6 @@ var mk20DealCmd = &cli.Command{ carFileSize := cctx.Uint64("car-size") - url, err := url.Parse(cctx.String("http-url")) - if err != nil { - return xerrors.Errorf("parsing http url: %w", err) - } - var headers http.Header for _, header := range cctx.StringSlice("http-headers") { @@ -1833,23 +1833,54 @@ var mk20DealCmd = &cli.Command{ if carFileSize == 0 { return xerrors.Errorf("size of car file cannot be 0") } - d = mk20.DataSource{ - PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceHTTP: &mk20.DataSourceHTTP{ - RawSize: carFileSize, - URLs: []mk20.HttpUrl{ - { - URL: url.String(), - Headers: headers, - Priority: 0, - Fallback: true, + + if !cctx.IsSet("http-url") { + if cctx.Bool("put") { + d = mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHttpPut: &mk20.DataSourceHttpPut{ + RawSize: carFileSize, + }, + } + } else { + d = mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceOffline: &mk20.DataSourceOffline{ + RawSize: carFileSize, + }, + } + } + } else { + url, err := url.Parse(cctx.String("http-url")) + if err != nil { + return xerrors.Errorf("parsing http url: %w", err) + } + d = mk20.DataSource{ + PieceCID: pieceCid, + Size: abi.PaddedPieceSize(pieceSize), + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + RawSize: carFileSize, + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Headers: headers, + Priority: 0, + Fallback: true, + }, }, }, - }, + } } } diff --git a/deps/deps.go b/deps/deps.go index 95866be9f..3ff89d2bc 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -355,7 +355,8 @@ Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, } if deps.IndexStore == nil { - deps.IndexStore, err = indexstore.NewIndexStore(strings.Split(cctx.String("db-host"), ","), cctx.Int("db-cassandra-port"), deps.Cfg) + deps.IndexStore = indexstore.NewIndexStore(strings.Split(cctx.String("db-host"), ","), cctx.Int("db-cassandra-port"), deps.Cfg) + err = deps.IndexStore.Start(cctx.Context, false) if err != nil { return xerrors.Errorf("failed to start index store: %w", err) } @@ -367,7 +368,7 @@ Get it with: jq .PrivateKey ~/.lotus-miner/keystore/MF2XI2BNNJ3XILLQOJUXMYLUMU`, if deps.CachedPieceReader == nil { ppr := pieceprovider.NewPieceParkReader(deps.Stor, deps.Si) - deps.CachedPieceReader = cachedreader.NewCachedPieceReader(deps.DB, deps.SectorReader, ppr) + deps.CachedPieceReader = cachedreader.NewCachedPieceReader(deps.DB, deps.SectorReader, ppr, deps.IndexStore) } if deps.ServeChunker == nil { diff --git a/docker/piece-server/sample/mk20-random-deal.sh b/docker/piece-server/sample/mk20-random-deal.sh index 9bf11ce12..1ff612f9c 100755 --- a/docker/piece-server/sample/mk20-random-deal.sh +++ b/docker/piece-server/sample/mk20-random-deal.sh @@ -4,21 +4,34 @@ set -e ci="\e[3m" cn="\e[0m" -chunks="${1:-51200}" -links="${2:-100}" +offline="${1:=false}" +chunks="${2:-51200}" +links="${3:-100}" printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" - FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` -PAYLOAD_CID=$(find "$FILE" | xargs -I{} basename {} | sed 's/\.car//') - read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk12-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') + +mv $FILE /var/lib/curio-client/data/$COMMP_CID + miner_actor=$(lotus state list-miners | grep -v t01000) -################################################################################### -printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ ---http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID \ ---commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ ---contract-address 0xtest --contract-verify-method test\n\n${cn}" +if [ "$offline" == "true" ]; then + + ################################################################################### + printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ + --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --contract-address 0xtest --contract-verify-method test\n\n${cn}" + + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + +else + ################################################################################### + printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ + --http-url=http://piece-server:12320/pieces?id=$COMMP_CID \ + --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --contract-address 0xtest --contract-verify-method test\n\n${cn}" + + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test -sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$PAYLOAD_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test \ No newline at end of file +fi \ No newline at end of file diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 52d0e94f2..800e0f4c9 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -899,9 +899,10 @@ USAGE: sptool toolbox mk20-client command [command options] COMMANDS: - init Initialise curio mk12 client repo - deal Make a mk20 deal with Curio - help, h Shows a list of commands or help for one command + init Initialise curio mk12 client repo + deal Make a mk20 deal with Curio + aggregate Create a new aggregate from a list of CAR files + help, h Shows a list of commands or help for one command OPTIONS: --mk12-client-repo value repo directory for mk12 client (default: "~/.curio-client") [$CURIO_MK12_CLIENT_REPO] @@ -936,9 +937,28 @@ OPTIONS: --commp value commp of the CAR file --piece-size value size of the CAR file as a padded piece (default: 0) --duration value duration of the deal in epochs (default: 518400) - --verified whether the deal funds should come from verified client data-cap (default: false) + --contract-address value contract address of the deal + --contract-verify-method value contract verify method of the deal + --allocation value allocation id of the deal (default: 0) --indexing indicates that an deal should be indexed (default: true) --wallet value wallet address to be used to initiate the deal --announce indicates that deal should be announced to the IPNI(Network Indexer) (default: true) + --aggregate value aggregate file path for the deal + --put used HTTP put as data source (default: false) --help, -h show help ``` + +#### sptool toolbox mk20-client aggregate +``` +NAME: + sptool toolbox mk20-client aggregate - Create a new aggregate from a list of CAR files + +USAGE: + sptool toolbox mk20-client aggregate [command options] + +OPTIONS: + --files value [ --files value ] list of CAR files to aggregate + --piece-size value piece size of the aggregate (default: 0) + --out output the aggregate file (default: false) + --help, -h show help +``` diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index cc53ed08c..eca6f7a9c 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -224,6 +224,75 @@ INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('put', TRUE); +CREATE OR REPLACE FUNCTION process_offline_download( + _id TEXT, + _piece_cid TEXT, + _piece_size BIGINT +) RETURNS BOOLEAN AS $$ +DECLARE +_url TEXT; + _headers JSONB; + _raw_size BIGINT; + _deal_aggregation INT; + _piece_id BIGINT; + _ref_id BIGINT; +BEGIN + -- 1. Early exit if no offline match found + SELECT url, headers, raw_size + INTO _url, _headers, _raw_size + FROM market_mk20_offline_urls + WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size; + + IF NOT FOUND THEN + RETURN FALSE; + END IF; + + -- 2. Get deal_aggregation flag + SELECT deal_aggregation + INTO _deal_aggregation + FROM market_mk20_pipeline + WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size + LIMIT 1; + + -- 3. Look for existing piece + SELECT id + INTO _piece_id + FROM parked_pieces + WHERE piece_cid = _piece_cid AND piece_padded_size = _piece_size; + + -- 4. Insert piece if not found + IF NOT FOUND THEN + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES (_piece_cid, _piece_size, _raw_size, NOT (_deal_aggregation > 0)) + RETURNING id INTO _piece_id; + END IF; + + -- 5. Insert piece ref + INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + VALUES (_piece_id, _url, _headers, NOT (_deal_aggregation > 0)) + RETURNING ref_id INTO _ref_id; + + -- 6. Insert or update download pipeline with ref_id + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) + VALUES (_id, _piece_cid, _piece_size, ARRAY[_ref_id]) + ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + SET ref_ids = ( + SELECT ARRAY( + SELECT DISTINCT r + FROM unnest(market_mk20_download_pipeline.ref_ids || excluded.ref_ids) AS r + ) + ); + + -- 7. Mark the deal as started + UPDATE market_mk20_pipeline + SET started = TRUE + WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size AND started = FALSE; + + RETURN TRUE; +END; +$$ LANGUAGE plpgsql; + + diff --git a/itests/curio_test.go b/itests/curio_test.go index 2e09d553f..da6305628 100644 --- a/itests/curio_test.go +++ b/itests/curio_test.go @@ -80,7 +80,8 @@ func TestCurioHappyPath(t *testing.T) { defer db.ITestDeleteAll() - idxStore, err := indexstore.NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, config.DefaultCurioConfig()) + idxStore := indexstore.NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, config.DefaultCurioConfig()) + err = idxStore.Start(ctx, true) require.NoError(t, err) var titles []string diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index 16edd1018..b2da0cd52 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -12,12 +12,15 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/jellydator/ttlcache/v2" + "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/indexstore" ) var NoDealErr = errors.New("no deals found") @@ -36,13 +39,15 @@ type CachedPieceReader struct { sectorReader *pieceprovider.SectorReader pieceParkReader *pieceprovider.PieceParkReader + idxStor *indexstore.IndexStore + pieceReaderCacheMu sync.Mutex pieceReaderCache *ttlcache.Cache // Cache for successful readers (10 minutes with TTL extension) pieceErrorCacheMu sync.Mutex pieceErrorCache *ttlcache.Cache // Cache for errors (5 seconds without TTL extension) } -func NewCachedPieceReader(db *harmonydb.DB, sectorReader *pieceprovider.SectorReader, pieceParkReader *pieceprovider.PieceParkReader) *CachedPieceReader { +func NewCachedPieceReader(db *harmonydb.DB, sectorReader *pieceprovider.SectorReader, pieceParkReader *pieceprovider.PieceParkReader, idxStor *indexstore.IndexStore) *CachedPieceReader { prCache := ttlcache.NewCache() _ = prCache.SetTTL(PieceReaderCacheTTL) prCache.SetCacheSizeLimit(MaxCachedReaders) @@ -59,6 +64,7 @@ func NewCachedPieceReader(db *harmonydb.DB, sectorReader *pieceprovider.SectorRe pieceParkReader: pieceParkReader, pieceReaderCache: prCache, pieceErrorCache: errorCache, + idxStor: idxStor, } expireCallback := func(key string, reason ttlcache.EvictionReason, value interface{}) { @@ -216,8 +222,46 @@ func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, p return reader, abi.UnpaddedPieceSize(pieceData[0].PieceRawSize), nil } -func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { - cacheKey := pieceCid.String() +func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { + pieces, err := cpr.idxStor.FindPieceInAggregate(ctx, pieceCidV2) + if err != nil { + return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) + } + + if len(pieces) == 0 { + return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) + } + + for _, p := range pieces { + commp, err := commcidv2.CommPFromPCidV2(p) + if err != nil { + return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + reader, payloadSize, err := cpr.getPieceReaderFromPiecePark(ctx, commp.PCidV1(), commp.PieceInfo().Size) + if err != nil { + log.Warnw("failed to get piece reader from piece park", "piececid", commp.PCidV1(), "piece size", commp.PieceInfo().Size, "err", err) + reader, payloadSize, err = cpr.getPieceReaderFromSector(ctx, commp.PCidV1(), commp.PieceInfo().Size) + if err != nil { + log.Errorw("failed to get piece reader from sector", "piececid", commp.PCidV1(), "piece size", commp.PieceInfo().Size, "err", err) + continue + } + return reader, payloadSize, nil + } + return reader, payloadSize, nil + } + return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) +} + +func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { + cacheKey := pieceCidV2.String() + + commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) + if err != nil { + return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + + pieceCid := commp.PCidV1() + pieceSize := commp.PieceInfo().Size // First check if we have a cached error for this piece cpr.pieceErrorCacheMu.Lock() @@ -239,7 +283,7 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid // to the cache r = &cachedSectionReader{ cpr: cpr, - pieceCid: pieceCid, + pieceCid: pieceCidV2, ready: make(chan struct{}), refs: 1, } @@ -250,33 +294,37 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid readerCtx, readerCtxCancel := context.WithCancel(context.Background()) defer close(r.ready) - reader, size, err := cpr.getPieceReaderFromSector(readerCtx, pieceCid, pieceSize) + reader, size, err := cpr.getPieceReaderFromAggregate(readerCtx, pieceCid) if err != nil { - log.Warnw("failed to get piece reader from sector", "piececid", pieceCid, "piece size", pieceSize, "err", err) + log.Warnw("failed to get piece reader from aggregate", "piececid", pieceCid, "piece size", pieceSize, "err", err) - serr := err + aerr := err - // Try getPieceReaderFromPiecePark - reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCid, pieceSize) + reader, size, err = cpr.getPieceReaderFromSector(readerCtx, pieceCid, pieceSize) if err != nil { - log.Errorw("failed to get piece reader from piece park", "piececid", pieceCid, "piece size", pieceSize, "err", err) - - finalErr := fmt.Errorf("failed to get piece reader from sector or piece park: %w, %w", err, serr) - - // Cache the error in the error cache - cpr.pieceErrorCacheMu.Lock() - _ = cpr.pieceErrorCache.Set(cacheKey, &cachedError{err: finalErr, pieceCid: pieceCid}) - cpr.pieceErrorCacheMu.Unlock() - - // Remove the failed reader from the main cache - cpr.pieceReaderCacheMu.Lock() - _ = cpr.pieceReaderCache.Remove(cacheKey) - cpr.pieceReaderCacheMu.Unlock() - - r.err = finalErr - readerCtxCancel() - - return nil, 0, finalErr + log.Warnw("failed to get piece reader from sector", "piececid", pieceCid, "piece size", pieceSize, "err", err) + serr := err + // Try getPieceReaderFromPiecePark + reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCid, pieceSize) + if err != nil { + log.Errorw("failed to get piece reader from piece park", "piececid", pieceCid, "piece size", pieceSize, "err", err) + finalErr := fmt.Errorf("failed to get piece reader from aggregate, sector or piece park: %w, %w, %w", aerr, serr, err) + + // Cache the error in the error cache + cpr.pieceErrorCacheMu.Lock() + _ = cpr.pieceErrorCache.Set(cacheKey, &cachedError{err: finalErr, pieceCid: pieceCid}) + cpr.pieceErrorCacheMu.Unlock() + + // Remove the failed reader from the main cache + cpr.pieceReaderCacheMu.Lock() + _ = cpr.pieceReaderCache.Remove(cacheKey) + cpr.pieceReaderCacheMu.Unlock() + + r.err = finalErr + readerCtxCancel() + + return nil, 0, finalErr + } } } diff --git a/lib/testutils/testutils.go b/lib/testutils/testutils.go index 30b6f8e3e..9ca88c8a6 100644 --- a/lib/testutils/testutils.go +++ b/lib/testutils/testutils.go @@ -270,7 +270,7 @@ func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggre defer os.Remove(f.Name()) } else { cn := path.Join(p, pcid.String()) - defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", cn)) + defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", cn)) //nolint:errcheck } return pcid, abi.PaddedPieceSize(paddedPieceSize), nil diff --git a/market/indexstore/create.cql b/market/indexstore/cql/0001_create.cql similarity index 100% rename from market/indexstore/create.cql rename to market/indexstore/cql/0001_create.cql diff --git a/market/indexstore/cql/0002_piece_index.cql b/market/indexstore/cql/0002_piece_index.cql new file mode 100644 index 000000000..b8bc6868b --- /dev/null +++ b/market/indexstore/cql/0002_piece_index.cql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS PieceToAggregatePiece ( + PieceCid BLOB, + AggregatePieceCid BLOB, + UnpaddedOffset BIGINT, + UnpaddedLength BIGINT, + PRIMARY KEY (PieceCid, AggregatePieceCid) +); \ No newline at end of file diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index c3dcaa66b..49dd463fd 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -2,9 +2,12 @@ package indexstore import ( "context" + "embed" _ "embed" "errors" "fmt" + "math/rand" + "strconv" "strings" "time" @@ -20,8 +23,8 @@ import ( const keyspace = "curio" -//go:embed create.cql -var createCQL string +//go:embed cql/*.cql +var cqlFiles embed.FS var log = logging.Logger("indexstore") @@ -71,35 +74,46 @@ func isNotFoundErr(err error) bool { return strings.Contains(strings.ToLower(err.Error()), "not found") } -func NewIndexStore(hosts []string, port int, cfg *config.CurioConfig) (*IndexStore, error) { - if len(hosts) == 0 { - return nil, xerrors.Errorf("no hosts provided for cassandra") - } - +func NewIndexStore(hosts []string, port int, cfg *config.CurioConfig) *IndexStore { cluster := gocql.NewCluster(hosts...) cluster.Timeout = 5 * time.Minute cluster.Consistency = gocql.One cluster.NumConns = cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency * 8 cluster.Port = port - store := &IndexStore{ + return &IndexStore{ cluster: cluster, settings: settings{ InsertBatchSize: cfg.Market.StorageMarketConfig.Indexing.InsertBatchSize, InsertConcurrency: cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency, }, } +} + +type ITestID string - return store, store.Start(context.Background()) +// ItestNewID see ITestWithID doc +func ITestNewID() ITestID { + return ITestID(strconv.Itoa(rand.Intn(99999))) } -func (i *IndexStore) Start(ctx context.Context) error { +func (i *IndexStore) Start(ctx context.Context, test bool) error { + if len(i.cluster.Hosts) == 0 { + return xerrors.Errorf("no hosts provided for cassandra") + } + + keyspaceName := keyspace + if test { + id := ITestNewID() + keyspaceName = fmt.Sprintf("test%s", id) + } + // Create Cassandra keyspace session, err := i.cluster.CreateSession() if err != nil { return xerrors.Errorf("creating cassandra session: %w", err) } - query := `CREATE KEYSPACE IF NOT EXISTS ` + keyspace + + query := `CREATE KEYSPACE IF NOT EXISTS ` + keyspaceName + ` WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 }` err = session.Query(query).WithContext(ctx).Exec() if err != nil { @@ -115,16 +129,32 @@ func (i *IndexStore) Start(ctx context.Context) error { return xerrors.Errorf("creating cassandra session: %w", err) } - lines := strings.Split(createCQL, ";") - for _, line := range lines { - line = strings.Trim(line, "\n \t") - if line == "" { + entries, err := cqlFiles.ReadDir("cql") + if err != nil { + log.Fatalf("failed to read embedded directory: %v", err) + } + + for _, entry := range entries { + if entry.IsDir() { continue } - log.Debug(line) - err := session.Query(line).WithContext(ctx).Exec() + + data, err := cqlFiles.ReadFile("cql/" + entry.Name()) if err != nil { - return xerrors.Errorf("creating tables: executing\n%s\n%w", line, err) + log.Fatalf("failed to read file %s: %v", entry.Name(), err) + } + + lines := strings.Split(string(data), ";") + for _, line := range lines { + line = strings.Trim(line, "\n \t") + if line == "" { + continue + } + log.Debug(line) + err := session.Query(line).WithContext(ctx).Exec() + if err != nil { + return xerrors.Errorf("creating tables: executing\n%s\n%w", line, err) + } } } @@ -406,3 +436,66 @@ func (i *IndexStore) CheckHasPiece(ctx context.Context, piecev2 cid.Cid) (bool, return len(hashes) > 0, nil } + +func (i *IndexStore) InsertAggregateIndex(ctx context.Context, aggregatePieceCid cid.Cid, records []Record) error { + insertAggregateIndex := `INSERT INTO PieceToAggregatePiece (PieceCid, AggregatePieceCid, UnpaddedOffset, UnpaddedLength) VALUES (?, ?, ?, ?)` + aggregatePieceCidBytes := aggregatePieceCid.Bytes() + var batch *gocql.Batch + batchSize := i.settings.InsertBatchSize + + if len(records) == 0 { + return xerrors.Errorf("no records to insert") + } + + for _, r := range records { + if batch == nil { + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: insertAggregateIndex, + Args: []interface{}{r.Cid.Bytes(), aggregatePieceCidBytes, r.Offset, r.Size}, + Idempotent: true, + }) + + if len(batch.Entries) >= batchSize { + if err := i.session.ExecuteBatch(batch); err != nil { + return xerrors.Errorf("executing batch insert for aggregate piece %s: %w", aggregatePieceCid, err) + } + batch = nil + } + } + + if batch != nil { + if len(batch.Entries) >= 0 { + if err := i.session.ExecuteBatch(batch); err != nil { + return xerrors.Errorf("executing batch insert for aggregate piece %s: %w", aggregatePieceCid, err) + } + } + } + + return nil +} + +func (i *IndexStore) FindPieceInAggregate(ctx context.Context, pieceCid cid.Cid) ([]cid.Cid, error) { + qry := `SELECT AggregatePieceCid FROM PieceToAggregatePiece WHERE PieceCid = ?` + iter := i.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() + var aggregatePieceCidBytes []cid.Cid + var r []byte + for iter.Scan(&r) { + c, err := cid.Cast(r) + if err != nil { + return nil, xerrors.Errorf("casting aggregate piece cid: %w", err) + } + aggregatePieceCidBytes = append(aggregatePieceCidBytes, c) + + r = make([]byte, 0) + } + if err := iter.Close(); err != nil { + return nil, xerrors.Errorf("iterating aggregate piece cid (P:0x%02x): %w", pieceCid.Bytes(), err) + } + if len(aggregatePieceCidBytes) == 0 { + return nil, nil + } + return aggregatePieceCidBytes, nil +} diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index 66b6424c0..79ee4fc4d 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -5,7 +5,6 @@ import ( "io" "os" "testing" - "time" carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" @@ -32,7 +31,8 @@ func TestNewIndexStore(t *testing.T) { ctx := context.Background() cfg := config.DefaultCurioConfig() - idxStore, err := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) + idxStore := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) + err := idxStore.Start(ctx, true) require.NoError(t, err) // Create a car file and calculate commP @@ -42,7 +42,7 @@ func TestNewIndexStore(t *testing.T) { _ = os.RemoveAll(dir) }() - rf, err := testutils.CreateRandomTmpFile(dir, time.Now().Unix(), 8000000) + rf, err := testutils.CreateRandomTmpFile(dir, 8000000) require.NoError(t, err) caropts := []carv2.Option{ @@ -117,6 +117,9 @@ func TestNewIndexStore(t *testing.T) { err = idxStore.RemoveIndexes(ctx, pcids[0].PieceCidV2) require.NoError(t, err) + err = idxStore.session.Query("SELECT * FROM PieceToAggregatePiece").Exec() + require.NoError(t, err) + // Drop the tables err = idxStore.session.Query("DROP TABLE PayloadToPieces").Exec() require.NoError(t, err) diff --git a/market/ipni/chunker/serve-chunker.go b/market/ipni/chunker/serve-chunker.go index a1dd60fed..9fcaf5f7b 100644 --- a/market/ipni/chunker/serve-chunker.go +++ b/market/ipni/chunker/serve-chunker.go @@ -226,7 +226,7 @@ func (p *ServeChunker) reconstructChunkFromCar(ctx context.Context, chunk, piece pi := commp.PieceInfo() - reader, _, err := p.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) + reader, _, err := p.cpr.GetSharedPieceReader(ctx, piecev2) defer func(reader storiface.Reader) { _ = reader.Close() }(reader) diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index d5f9ada79..41abe2811 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -5,6 +5,7 @@ import ( "context" _ "embed" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -15,6 +16,7 @@ import ( "github.com/go-chi/httprate" logging "github.com/ipfs/go-log/v2" "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" @@ -122,7 +124,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { // mk20status handles HTTP requests to retrieve the status of a deal using its ID, responding with deal status or appropriate error codes. func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { // Extract id from the URL - idStr := chi.URLParam(r, "id") + idStr := r.URL.Query().Get("id") if idStr == "" { log.Errorw("missing id in url", "url", r.URL) http.Error(w, "missing id in url", http.StatusBadRequest) @@ -161,6 +163,11 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt var contracts mk20.SupportedContracts err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM contracts") if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + log.Errorw("no supported contracts found") + http.Error(w, "no supported contracts found", http.StatusNotFound) + return + } log.Errorw("failed to get supported contracts", "err", err) w.WriteHeader(http.StatusInternalServerError) return @@ -183,7 +190,7 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt // mk20UploadDealData handles uploading deal data to the server using a PUT request with specific validations and streams directly to the logic. func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Request) { // Extract id from the URL - idStr := chi.URLParam(r, "id") + idStr := r.URL.Query().Get("id") if idStr == "" { log.Errorw("missing id in url", "url", r.URL) w.WriteHeader(http.StatusBadRequest) diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md index e39d83d26..d4802b744 100644 --- a/market/mk20/http/info.md +++ b/market/mk20/http/info.md @@ -97,7 +97,7 @@ DataSource represents the source of piece data, including metadata and optional | Field | Type | Tag | Description | |-------|------|-----|-------------| | PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | PieceCID represents the unique identifier for a piece of data, stored as a CID object. | -| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"size" | Size represents the size of the padded piece in the data source. | +| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"piece_size" | Size represents the size of the padded piece in the data source. | | Format | [mk20.PieceDataFormat](#piecedataformat) | json:"format" | Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. | | SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"source_http" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | | SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"source_aggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index b8b153325..866766324 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -89,6 +89,8 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti return ret } + log.Debugw("deal validated", "deal", deal.Identifier.String()) + return m.processDDODeal(ctx, deal) } @@ -99,6 +101,9 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje log.Errorw("deal rejected", "deal", deal, "error", err) return rejection } + + log.Debugw("deal sanitized", "deal", deal.Identifier.String()) + if rejection != nil { return rejection } @@ -117,6 +122,8 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje return ret } + log.Debugw("deal ID found", "deal", deal.Identifier.String(), "id", id) + // TODO: Backpressure, client filter comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { @@ -158,6 +165,8 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje } } + log.Debugw("deal inserted in DB", "deal", deal.Identifier.String()) + return &ProviderDealRejectionInfo{ HTTPCode: http.StatusOK, } diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index 39e719176..772172ca4 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "strings" "time" "github.com/oklog/ulid" @@ -15,6 +16,9 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/dealdata" @@ -25,7 +29,7 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { var dealError sql.NullString - err := m.db.QueryRow(ctx, `SELECT error FROM market_mk20_pipeline WHERE id = $1)`, id.String()).Scan(&dealError) + err := m.db.QueryRow(ctx, `SELECT error FROM market_mk20_deal WHERE id = $1;`, id.String()).Scan(&dealError) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return &DealStatus{ @@ -142,7 +146,7 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons } err := m.db.Select(ctx, &waitingDeal, `SELECT started_put, start_time from market_mk20_pipeline_waiting - WHERE waiting_for_data = TRUE AND id = $1)`, id.String()) + WHERE waiting_for_data = TRUE AND id = $1`, id.String()) if err != nil { log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) @@ -154,8 +158,10 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons http.Error(w, "", http.StatusNotFound) } - if waitingDeal[0].Started && waitingDeal[0].StartTime.Add(m.cfg.HTTP.ReadTimeout).Before(time.Now()) { - http.Error(w, "another /PUT request is in progress for this deal", http.StatusConflict) + if waitingDeal[0].Started { + if waitingDeal[0].StartTime.Add(m.cfg.HTTP.ReadTimeout).Before(time.Now()) { + http.Error(w, "another /PUT request is in progress for this deal", http.StatusConflict) + } } // TODO: Rethink how to ensure only 1 process per deal for /PUT @@ -198,21 +204,41 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons } }() + cp := new(commp.Calc) + // Function to write data into StashStore and calculate commP writeFunc := func(f *os.File) error { limitedReader := io.LimitReader(data, int64(rawSize+1)) // +1 to detect exceeding the limit + wr := io.MultiWriter(f, cp) - n, err := io.Copy(f, limitedReader) + size, err := io.Copy(wr, limitedReader) if err != nil { return fmt.Errorf("failed to read and write piece data: %w", err) } - if n > int64(deal.Data.Size) { + if size > int64(deal.Data.Size) { return fmt.Errorf("piece data exceeds the maximum allowed size") } - if int64(rawSize) != n { - return fmt.Errorf("raw size does not match with uploaded data: %w", err) + if int64(rawSize) != size { + return fmt.Errorf("deal raw size %d does not match with uploaded data size %d", rawSize, size) + } + + digest, pieceSize, err := cp.Digest() + if err != nil { + return fmt.Errorf("failed to calculate commP: %w", err) + } + + pieceCIDComputed, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return fmt.Errorf("failed to calculate piece CID: %w", err) + } + if !pieceCIDComputed.Equals(deal.Data.PieceCID) { + return fmt.Errorf("calculated piece CID %s does not match with deal piece CID %s", pieceCIDComputed.String(), deal.Data.PieceCID.String()) + } + + if abi.PaddedPieceSize(pieceSize) != deal.Data.Size { + return fmt.Errorf("calculated piece size %d does not match with deal piece size %d", pieceSize, deal.Data.Size) } return nil @@ -222,10 +248,24 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons stashID, err := m.stor.StashCreate(ctx, int64(deal.Data.Size), writeFunc) if err != nil { if err.Error() == "piece data exceeds the maximum allowed size" { - http.Error(w, err.Error(), http.StatusRequestEntityTooLarge) + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, "piece data exceeds the maximum allowed size", http.StatusRequestEntityTooLarge) + return + } else if strings.Contains(err.Error(), "does not match with uploaded data") { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) return - } else if err.Error() == "raw size does not match with uploaded data" { - http.Error(w, err.Error(), http.StatusRequestEntityTooLarge) + } else if strings.Contains(err.Error(), "failed to calculate piece CID") { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, "Failed to calculate piece CID", http.StatusInternalServerError) + return + } else if strings.Contains(err.Error(), "calculated piece CID does not match with uploaded data") { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) + return + } else if strings.Contains(err.Error(), "calculated piece size does not match with uploaded data") { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) return } else { log.Errorw("Failed to store piece data in StashStore", "error", err) @@ -290,20 +330,18 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons allocationID = nil } - var aggregation interface{} + aggregation := 0 if dealdata.Format.Aggregate != nil { - aggregation = dealdata.Format.Aggregate.Type - } else { - aggregation = nil + aggregation = int(dealdata.Format.Aggregate.Type) } n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( id, sp_id, contract, client, piece_cid, piece_size, raw_size, offline, indexing, announce, - allocation_id, duration, piece_aggregation, started) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE)`, + allocation_id, duration, piece_aggregation, started, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE, TRUE)`, dealID, spid, ddo.ContractAddress, ddo.Client.String(), dealdata.PieceCID.String(), - dealdata.Size, int64(dealdata.SourceHTTP.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, + dealdata.Size, int64(dealdata.SourceHttpPut.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, aggregation) if err != nil { return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) @@ -320,10 +358,22 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons return true, nil // Commit the transaction }, harmonydb.OptionRetry()) - if err != nil || !comm { - // Remove the stash file as the transaction failed - _ = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to process piece upload", "Deal", id, "error", err) + http.Error(w, "Failed to process piece upload", http.StatusInternalServerError) + err = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to remove stash file", "Deal", id, "error", err) + } + return + } + + if !comm { http.Error(w, "Failed to process piece upload", http.StatusInternalServerError) + err = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to remove stash file", "Deal", id, "error", err) + } return } diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 17c67747f..e968620b1 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -481,6 +481,14 @@ func (d *DBDeal) ToDeal() (*Deal, error) { ds.SourceOffline = &so } + if len(d.SourceHttpPut) > 0 && string(d.SourceHttpPut) != "null" { + var shp DataSourceHttpPut + if err := json.Unmarshal(d.SourceHttpPut, &shp); err != nil { + return nil, fmt.Errorf("unmarshal source_http_put: %w", err) + } + ds.SourceHttpPut = &shp + } + if len(d.DDOv1) > 0 && string(d.DDOv1) != "null" { if err := json.Unmarshal(d.DDOv1, &products.DDOV1); err != nil { return nil, fmt.Errorf("unmarshal ddov1: %w", err) diff --git a/market/retrieval/piecehandler.go b/market/retrieval/piecehandler.go index 3449b1b8b..931b0eb5b 100644 --- a/market/retrieval/piecehandler.go +++ b/market/retrieval/piecehandler.go @@ -13,7 +13,6 @@ import ( "go.opencensus.io/stats" "github.com/filecoin-project/curio/lib/cachedreader" - "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/retrieval/remoteblockstore" ) @@ -45,18 +44,8 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { return } - commp, err := commcidv2.CommPFromPCidV2(pieceCid) - if err != nil { - log.Errorf("parsing piece CID '%s': %s", pieceCidStr, err.Error()) - w.WriteHeader(http.StatusBadRequest) - stats.Record(ctx, remoteblockstore.HttpPieceByCid400ResponseCount.M(1)) - return - } - - pi := commp.PieceInfo() - // Get a reader over the piece - reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) + reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pieceCid) if err != nil { log.Errorf("server error getting content for piece CID %s: %s", pieceCid, err) if errors.Is(err, cachedreader.NoDealErr) { diff --git a/market/retrieval/remoteblockstore/remoteblockstore.go b/market/retrieval/remoteblockstore/remoteblockstore.go index 9551c7964..82a35dde1 100644 --- a/market/retrieval/remoteblockstore/remoteblockstore.go +++ b/market/retrieval/remoteblockstore/remoteblockstore.go @@ -19,7 +19,6 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/cachedreader" - "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" ) @@ -116,13 +115,7 @@ func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, var merr error for _, piece := range pieces { data, err := func() ([]byte, error) { - // Get a reader over the piece data - commp, err := commcidv2.CommPFromPCidV2(piece.PieceCidV2) - if err != nil { - return nil, fmt.Errorf("getting commP from piece cid v2 %s: %w", piece.PieceCidV2.String(), err) - } - pi := commp.PieceInfo() - reader, _, err := ro.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) + reader, _, err := ro.cpr.GetSharedPieceReader(ctx, piece.PieceCidV2) if err != nil { return nil, fmt.Errorf("getting piece reader: %w", err) } diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index 0cf946987..a28287024 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -181,12 +181,17 @@ func (s *PipelineGC) cleanupMK20DealPipeline() error { _, err := s.db.Exec(ctx, `DELETE FROM market_mk20_offline_urls WHERE id IN ( - SELECT id FROM market_mk20_pipeline WHERE complete = TRUE - ); - - DELETE FROM market_mk20_pipeline - WHERE complete = TRUE; - `) + SELECT id FROM market_mk20_pipeline WHERE complete = TRUE)`) + if err != nil { + return xerrors.Errorf("failed to clean up offline urls: %w", err) + } + _, err = s.db.Exec(ctx, `DELETE FROM market_mk20_download_pipeline + WHERE id IN ( + SELECT id FROM market_mk20_pipeline WHERE complete = TRUE)`) + if err != nil { + return xerrors.Errorf("failed to clean up download pipeline: %w", err) + } + _, err = s.db.Exec(ctx, `DELETE FROM market_mk20_pipeline WHERE complete = TRUE;`) if err != nil { return xerrors.Errorf("failed to clean up sealed deals: %w", err) } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index d1b8e2b30..7391dd553 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -198,7 +198,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do } pc2 := commp.PCidV2() - reader, _, err := i.cpr.GetSharedPieceReader(ctx, pieceCid, task.Size) + reader, _, err := i.cpr.GetSharedPieceReader(ctx, pc2) if err != nil { return false, xerrors.Errorf("getting piece reader: %w", err) @@ -223,8 +223,10 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return i.indexStore.AddIndex(ctx, pc2, recs) }) + var aggidx map[cid.Cid][]datasegment.SegmentDesc + if task.Mk20 && len(subPieces) > 0 { - blocks, interrupted, err = IndexAggregate(reader, task.Size, subPieces, recs, addFail) + blocks, aggidx, interrupted, err = IndexAggregate(pc2, reader, task.Size, subPieces, recs, addFail) } else { blocks, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) } @@ -248,6 +250,29 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do log.Infof("Indexing deal %s took %0.3f seconds", task.UUID, time.Since(startTime).Seconds()) + // Save aggregate index if present + for k, v := range aggidx { + var idxrecs []indexstore.Record + for _, r := range v { + pi := abi.PieceInfo{PieceCID: r.PieceCID(), Size: abi.PaddedPieceSize(r.Size)} + idxcommp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return false, xerrors.Errorf("getting piece commP: %w", err) + } + idxrecs = append(idxrecs, indexstore.Record{ + Cid: idxcommp.PCidV2(), + Offset: r.UnpaddedOffest(), + Size: r.UnpaddedLength(), + }) + } + if len(idxrecs) > 0 { + err = i.indexStore.InsertAggregateIndex(ctx, k, idxrecs) + if err != nil { + return false, xerrors.Errorf("inserting aggregate index: %w", err) + } + } + } + err = i.recordCompletion(ctx, task, taskID, true) if err != nil { return false, err @@ -411,38 +436,40 @@ type IndexReader interface { io.Reader } -func IndexAggregate( +func IndexAggregate(pieceCid cid.Cid, reader IndexReader, size abi.PaddedPieceSize, subPieces []mk20.PieceDataFormat, recs chan<- indexstore.Record, addFail <-chan struct{}, -) (int64, bool, error) { +) (int64, map[cid.Cid][]datasegment.SegmentDesc, bool, error) { dsis := datasegment.DataSegmentIndexStartOffset(size) if _, err := reader.Seek(int64(dsis), io.SeekStart); err != nil { - return 0, false, xerrors.Errorf("seeking to data segment index start offset: %w", err) + return 0, nil, false, xerrors.Errorf("seeking to data segment index start offset: %w", err) } idata, err := parseDataSegmentIndex(reader) if err != nil { - return 0, false, xerrors.Errorf("parsing data segment index: %w", err) + return 0, nil, false, xerrors.Errorf("parsing data segment index: %w", err) } if len(idata.Entries) == 0 { - return 0, false, xerrors.New("no data segment index entries") + return 0, nil, false, xerrors.New("no data segment index entries") } valid := validateSegments(idata.Entries) if len(valid) == 0 { - return 0, false, xerrors.New("no valid data segment index entries") + return 0, nil, false, xerrors.New("no valid data segment index entries") } + aggidx := make(map[cid.Cid][]datasegment.SegmentDesc) + log.Infow("Indexing aggregate", "piece_size", size, "num_chunks", len(valid), "num_sub_pieces", len(subPieces)) var haveSubPieces bool if len(subPieces) > 0 { if len(valid) != len(subPieces) { - return 0, false, xerrors.Errorf("expected %d data segment index entries, got %d", len(subPieces), len(idata.Entries)) + return 0, nil, false, xerrors.Errorf("expected %d data segment index entries, got %d", len(subPieces), len(idata.Entries)) } haveSubPieces = true } @@ -456,6 +483,13 @@ func IndexAggregate( strt := entry.UnpaddedOffest() leng := entry.UnpaddedLength() sectionReader := io.NewSectionReader(reader, int64(strt), int64(leng)) + pi := abi.PieceInfo{PieceCID: entry.PieceCID(), Size: abi.PaddedPieceSize(entry.Size)} + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return 0, nil, false, xerrors.Errorf("getting piece commP: %w", err) + } + + var idx map[cid.Cid][]datasegment.SegmentDesc b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) totalBlocks += b @@ -464,31 +498,34 @@ func IndexAggregate( if strings.Contains(err.Error(), "invalid car version") { if haveSubPieces { if subPieces[j].Car != nil { - return 0, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) + return 0, aggidx, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) } if subPieces[j].Raw != nil { continue } if subPieces[j].Aggregate != nil { - b, inter, err = IndexAggregate(sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) + b, idx, inter, err = IndexAggregate(commp.PCidV2(), sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) if err != nil { - return totalBlocks, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) + return totalBlocks, aggidx, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) } totalBlocks += b + for k, v := range idx { + aggidx[k] = append(aggidx[k], v...) + } } } else { continue } } - return totalBlocks, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) + return totalBlocks, aggidx, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) } if inter { - return totalBlocks, true, nil + return totalBlocks, aggidx, true, nil } } - return totalBlocks, false, nil + return totalBlocks, aggidx, false, nil } // recordCompletion add the piece metadata and piece deal to the DB and diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index da6dd8e61..bd3ae2349 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -115,7 +115,12 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, xerrors.Errorf("unmarshaling piece info: %w", err) } - reader, _, err := I.cpr.GetSharedPieceReader(ctx, pi.PieceCID, pi.Size) + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return false, xerrors.Errorf("getting piece commP: %w", err) + } + + reader, _, err := I.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) if err != nil { return false, xerrors.Errorf("getting piece reader: %w", err) @@ -144,11 +149,6 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b var subPieces []mk20.PieceDataFormat chk := chunker.NewInitialChunker() - commp, err := commcidv2.CommPFromPieceInfo(pi) - if err != nil { - return false, xerrors.Errorf("getting piece commP: %w", err) - } - eg.Go(func() error { defer close(addFail) for rec := range recs { @@ -182,7 +182,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { subPieces = deal.Data.Format.Aggregate.Sub - _, interrupted, err = IndexAggregate(reader, pi.Size, subPieces, recs, addFail) + _, _, interrupted, err = IndexAggregate(commp.PCidV2(), reader, pi.Size, subPieces, recs, addFail) } } diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index 93a71aabb..3e2daab88 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -32,6 +32,7 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/cachedreader" "github.com/filecoin-project/curio/lib/chainsched" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/curio/lib/proof" "github.com/filecoin-project/curio/pdp/contract" @@ -392,7 +393,12 @@ func (p *ProveTask) genSubrootMemtree(ctx context.Context, subrootCid string, su return nil, xerrors.Errorf("subroot size exceeds maximum: %d", subrootSize) } - subrootReader, unssize, err := p.cpr.GetSharedPieceReader(ctx, subrootCidObj, subrootSize) + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: subrootCidObj, Size: subrootSize}) + if err != nil { + return nil, xerrors.Errorf("failed to get piece commitment: %w", err) + } + + subrootReader, unssize, err := p.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) if err != nil { return nil, xerrors.Errorf("failed to get subroot reader: %w", err) } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 5e17df715..ac09a7fb6 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -257,7 +257,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 allocation_id, duration, piece_aggregation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, dealID, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), - data.Size, int64(data.SourceHTTP.RawSize), true, ddo.Indexing, ddo.AnnounceToIPNI, + data.Size, int64(data.SourceOffline.RawSize), true, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, aggregation) if err != nil { return xerrors.Errorf("inserting mk20 pipeline: %w", err) @@ -345,131 +345,6 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 } } - //existingCount = 0 - //for _, v := range existing { - // if v != nil { - // existingCount++ - // } - //} - // - //log.Infow("Initial Existing after first pass", "Count", existingCount) - // - //piBatch := &pgx.Batch{} - //piBatchSize := 10000 - //for k, v := range existing { - // if v == nil { - // piBatch.Queue(`WITH inserted_piece AS ( - // INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - // VALUES ($1, $2, $3, FALSE) - // ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING - // RETURNING id - // ), - // selected_piece AS ( - // SELECT COALESCE( - // (SELECT id FROM inserted_piece), - // (SELECT id FROM parked_pieces - // WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = FALSE AND cleanup_task_id IS NULL) - // ) AS id - // ), - // inserted_ref AS ( - // INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) - // SELECT id, $4, $5, FALSE FROM selected_piece - // RETURNING ref_id - // ) - // INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - // VALUES ($6, $1, $2, ARRAY[(SELECT ref_id FROM inserted_ref)]) - // ON CONFLICT (id, piece_cid, piece_size) DO UPDATE - // SET ref_ids = array_append( - // market_mk20_download_pipeline.ref_ids, - // (SELECT ref_id FROM inserted_ref) - // ) - // WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, - // k.PieceCID.String(), k.Size) - // if piBatch.Len() > piBatchSize { - // res := tx.SendBatch(ctx, piBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing parked piece insert batch: %w", err) - // } - // piBatch = &pgx.Batch{} - // } - // } - //} - // - //if piBatch.Len() > 0 { - // res := tx.SendBatch(ctx, piBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing parked piece insert batch: %w", err) - // } - //} - // - //existingCount = 0 - //for _, v := range existing { - // if v != nil { - // existingCount++ - // } - //} - // - //log.Infow("Initial Existing after second pass", "Count", existingCount) - // - //prBatch := &pgx.Batch{} - //prBatchSize := 10000 - // - //for k, v := range existing { - // if v == nil { - // return xerrors.Errorf("missing parked piece for %s", k.PieceCID.String()) - // } - // urls := toDownload[downloadkey{PieceCID: k.PieceCID, Size: k.Size}] - // for _, src := range urls { - // headers, err := json.Marshal(src.Headers) - // if err != nil { - // return xerrors.Errorf("marshal headers: %w", err) - // } - // prBatch.Queue(`WITH inserted_ref AS ( - // INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) - // VALUES ($1, $2, $3, FALSE) - // RETURNING ref_id - // ) - // INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - // VALUES ($4, $5, $6, ARRAY[(SELECT ref_id FROM inserted_ref)])`, - // *v, src.URL, headers, k.ID, k.PieceCID.String(), k.Size) - // } - // - // if prBatch.Len() > 0 { - // res := tx.SendBatch(ctx, prBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing parked piece ref insert batch: %w", err) - // } - // } - //} - // - //if prBatch.Len() > prBatchSize { - // res := tx.SendBatch(ctx, prBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing parked piece ref insert batch: %w", err) - // } - // prBatch = &pgx.Batch{} - //} - - //mdBatch := &pgx.Batch{} - //mdBatchSize := 20000 - //for k, v := range downloadMap { - // mdBatch.Queue(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - // k.ID, k.PieceCID.String(), k.Size, v) - // if mdBatch.Len() > mdBatchSize { - // res := tx.SendBatch(ctx, mdBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) - // } - // mdBatch = &pgx.Batch{} - // } - //} - //if mdBatch.Len() > 0 { - // res := tx.SendBatch(ctx, mdBatch) - // if err := res.Close(); err != nil { - // return xerrors.Errorf("closing mk20 download pipeline insert batch: %w", err) - // } - //} - pBatch := &pgx.Batch{} pBatchSize := 4000 for i, piece := range deal.Data.SourceAggregate.Pieces { @@ -606,19 +481,11 @@ func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK2 } return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) } - _, err = tx.Exec(` - DELETE FROM parked_piece_refs - WHERE ref_id IN ( - SELECT unnest(dp.ref_ids) - FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 - AND dp.piece_cid = $2 - AND dp.piece_size = $3 - ) - AND ref_id != $4; - `, piece.ID, piece.PieceCID, piece.PieceSize, refid) + + _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3`, + piece.ID, piece.PieceCID, piece.PieceSize) if err != nil { - return false, xerrors.Errorf("failed to delete parked piece refs: %w", err) + return false, xerrors.Errorf("failed to delete piece from download table: %w", err) } pieceIDUrl := url.URL{ @@ -651,68 +518,10 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie if piece.Offline && !piece.Downloaded && !piece.Started { comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { var updated bool - err = tx.QueryRow(` - WITH offline_match AS ( - SELECT url, headers, raw_size - FROM market_mk20_offline_urls - WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 - ), - existing_piece AS ( - SELECT id AS piece_id - FROM parked_pieces - WHERE piece_cid = $2 AND piece_padded_size = $3 - ), - inserted_piece AS ( - INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - SELECT $2, $3, o.raw_size, NOT (p.deal_aggregation > 0) - FROM offline_match o, market_mk20_pipeline p - WHERE p.id = $1 AND p.piece_cid = $2 AND p.piece_size = $3 - AND NOT EXISTS (SELECT 1 FROM existing_piece) - RETURNING id AS piece_id - ), - selected_piece AS ( - SELECT piece_id FROM existing_piece - UNION ALL - SELECT piece_id FROM inserted_piece - ), - inserted_refs AS ( - INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) - SELECT - s.piece_id, - o.url, - o.headers, - NOT (p.deal_aggregation > 0) - FROM selected_piece s - JOIN offline_match o ON true - JOIN market_mk20_pipeline p ON p.id = $1 AND p.piece_cid = $2 AND p.piece_size = $3 - RETURNING ref_id - ), - upsert_pipeline AS ( - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - SELECT $1, $2, $3, array_agg(ref_id) - FROM inserted_refs - ON CONFLICT (id, piece_cid, piece_size) DO UPDATE - SET ref_ids = ( - SELECT array( - SELECT DISTINCT unnest(dp.ref_ids) || unnest(EXCLUDED.ref_ids) - ) - ) - FROM market_mk20_download_pipeline dp - WHERE dp.id = EXCLUDED.id AND dp.piece_cid = EXCLUDED.piece_cid AND dp.piece_size = EXCLUDED.piece_size - RETURNING id - ), - mark_started AS ( - UPDATE market_mk20_pipeline - SET started = TRUE - WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 - AND EXISTS (SELECT 1 FROM offline_match) - RETURNING id - ) - SELECT EXISTS (SELECT 1 FROM mark_started); - `, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&updated) + err = tx.QueryRow(`SELECT process_offline_download($1, $2, $3)`, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&updated) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { - return false, xerrors.Errorf("failed to update the pipeline for deal %s: %w", piece.ID, err) + return false, xerrors.Errorf("failed to start download for offline deal %s: %w", piece.ID, err) } } @@ -810,19 +619,17 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie ON CONFLICT (id, piece_cid, piece_size) DO UPDATE SET ref_ids = ( SELECT array( - SELECT DISTINCT unnest(dp.ref_ids) || unnest(EXCLUDED.ref_ids) + SELECT DISTINCT r + FROM unnest(market_mk20_download_pipeline.ref_ids || excluded.ref_ids) AS r ) ) - FROM market_mk20_download_pipeline dp - WHERE dp.id = EXCLUDED.id AND dp.piece_cid = EXCLUDED.piece_cid AND dp.piece_size = EXCLUDED.piece_size - ), - mark_started AS ( - UPDATE market_mk20_pipeline - SET started = TRUE - WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND started = FALSE - )`, piece.ID, piece.PieceCID, piece.PieceSize, rUrl, hdrs, rawSize) + ) + UPDATE market_mk20_pipeline + SET started = TRUE + WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND started = FALSE;`, + piece.ID, piece.PieceCID, piece.PieceSize, rawSize, urlString, hdrs) if err != nil { - return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) + return false, xerrors.Errorf("failed to start download for offline deal using PieceLocator: %w", err) } return true, nil @@ -978,6 +785,8 @@ func (d *CurioStorageDealMarket) processMK20DealAggregation(ctx context.Context) d.adders[pollerAggregate].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, err error) { n, err := tx.Exec(`UPDATE market_mk20_pipeline SET agg_task_id = $1 WHERE id = $2 + AND started = TRUE + AND downloaded = TRUE AND after_commp = TRUE AND aggregated = FALSE AND agg_task_id IS NULL`, id, deal.ID) From c80f1bfca8dd141e3a2e291e9b3f25781d40f605 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 28 May 2025 14:35:43 +0400 Subject: [PATCH 09/55] finish UI --- web/api/webrpc/market.go | 161 +++++-- web/api/webrpc/market_20.go | 447 ++++++++++++++++++ .../pages/mk12-deals/deal-pipelines.mjs | 4 +- web/static/pages/mk12-deals/index.html | 2 +- web/static/pages/mk12-deals/mk12-deals.mjs | 12 +- web/static/pages/mk20-deal/deal.mjs | 243 +++++----- web/static/pages/mk20-deal/index.html | 7 + web/static/pages/mk20/ddo-pipeline.mjs | 376 +++++++++++++++ web/static/pages/mk20/ddo.mjs | 13 +- web/static/pages/mk20/index.html | 50 +- web/static/pages/piece/piece-info.mjs | 152 +++--- web/static/ux/curio-ux.mjs | 4 +- 12 files changed, 1190 insertions(+), 281 deletions(-) create mode 100644 web/static/pages/mk20/ddo-pipeline.mjs diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 59f47cfa3..b76b4d2b7 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" "github.com/ipfs/go-cid" + "github.com/oklog/ulid" "github.com/samber/lo" "github.com/snadrus/must" "github.com/yugabyte/pgx/v5" @@ -742,13 +743,13 @@ func (a *WebRPC) PieceParkStates(ctx context.Context, pieceCID string) (*ParkedP // Query the parked_pieces table err = a.deps.DB.QueryRow(ctx, ` SELECT id, created_at, piece_cid, piece_padded_size, piece_raw_size, complete, task_id, cleanup_task_id - FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND piece_raw_size = $3 - `, pi.PieceCID.String(), pi.Size, commp.PayloadSize()).Scan( + FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 + `, pi.PieceCID.String(), pi.Size).Scan( &pps.ID, &pps.CreatedAt, &pps.PieceCID, &pps.PiecePaddedSize, &pps.PieceRawSize, &pps.Complete, &pps.TaskID, &pps.CleanupTaskID, ) if err != nil { - if err == pgx.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { return nil, nil } return nil, fmt.Errorf("failed to query parked piece: %w", err) @@ -887,14 +888,17 @@ type MK20DealPipeline struct { Sector sql.NullInt64 `db:"sector" json:"sector"` RegSealProof sql.NullInt64 `db:"reg_seal_proof" json:"reg_seal_proof"` SectorOffset sql.NullInt64 `db:"sector_offset" json:"sector_offset"` - Sealed sql.NullBool `db:"sealed" json:"sealed"` + Sealed bool `db:"sealed" json:"sealed"` IndexingCreatedAt sql.NullTime `db:"indexing_created_at" json:"indexing_created_at"` IndexingTaskId sql.NullInt64 `db:"indexing_task_id" json:"indexing_task_id"` - Indexed sql.NullBool `db:"indexed" json:"indexed"` + Indexed bool `db:"indexed" json:"indexed"` Complete bool `db:"complete" json:"complete"` CreatedAt time.Time `db:"created_at" json:"created_at"` + + Miner string `db:"-" json:"miner"` + PieceCidV2 string `db:"-" json:"piece_cid_v2"` } type PieceInfoMK12Deals struct { @@ -1041,22 +1045,6 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe pipelineMap[pipeline.UUID] = pipeline } - ret := &PieceDealDetailEntry{ - MK12: make([]PieceInfoMK12Deals, len(mk12Deals)), - } - - for _, deal := range mk12Deals { - entry := PieceInfoMK12Deals{ - Deal: deal, - } - if pipeline, exists := pipelineMap[deal.UUID]; exists { - entry.Pipeline = &pipeline - } else { - entry.Pipeline = nil // Pipeline may not exist for processed and active deals - } - ret.MK12 = append(ret.MK12, entry) - } - var mk20Deals []*mk20.DBDeal err = a.deps.DB.Select(ctx, &mk20Deals, `SELECT id, @@ -1088,7 +1076,7 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe } } - var mk20Pipelines []*MK20DealPipeline + var mk20Pipelines []MK20DealPipeline err = a.deps.DB.Select(ctx, &mk20Pipelines, ` SELECT created_at, @@ -1128,11 +1116,25 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe } mk20pipelineMap := make(map[string]MK20DealPipeline) - for _, pipeline := range mk20pipelineMap { + for _, pipeline := range mk20Pipelines { pipeline := pipeline mk20pipelineMap[pipeline.ID] = pipeline } + ret := &PieceDealDetailEntry{} + + for _, deal := range mk12Deals { + entry := PieceInfoMK12Deals{ + Deal: deal, + } + if pipeline, exists := pipelineMap[deal.UUID]; exists { + entry.Pipeline = &pipeline + } else { + entry.Pipeline = nil // Pipeline may not exist for processed and active deals + } + ret.MK12 = append(ret.MK12, entry) + } + for _, deal := range mk20deals { entry := PieceInfoMK20Deals{ Deal: deal, @@ -1158,7 +1160,104 @@ func firstOrZero[T any](a []T) T { return a[0] } -func (a *WebRPC) MK12DealPipelineRemove(ctx context.Context, uuid string) error { +func (a *WebRPC) DealPipelineRemove(ctx context.Context, id string) error { + _, err := ulid.Parse(id) + if err != nil { + _, err = uuid.Parse(id) + if err != nil { + return xerrors.Errorf("invalid pipeline id: %w", err) + } + return a.mk12DealPipelineRemove(ctx, id) + } + return a.mk20DealPipelineRemove(ctx, id) +} + +func (a *WebRPC) mk20DealPipelineRemove(ctx context.Context, id string) error { + _, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var pipelines []struct { + Url string `db:"url"` + Sector sql.NullInt64 `db:"sector"` + + CommpTaskID sql.NullInt64 `db:"commp_task_id"` + AggrTaskID sql.NullInt64 `db:"agg_task_id"` + IndexingTaskID sql.NullInt64 `db:"indexing_task_id"` + } + + err = tx.Select(&pipelines, `SELECT url, sector, commp_task_id, agg_task_id, indexing_task_id + FROM market_mk20_pipeline WHERE id = $1`, id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("no deal pipeline found with id %s", id) + } + return false, err + } + + if len(pipelines) == 0 { + return false, fmt.Errorf("no deal pipeline found with id %s", id) + } + + // Collect non-null task IDs + var taskIDs []int64 + for _, pipeline := range pipelines { + if pipeline.CommpTaskID.Valid { + taskIDs = append(taskIDs, pipeline.CommpTaskID.Int64) + } + if pipeline.AggrTaskID.Valid { + taskIDs = append(taskIDs, pipeline.AggrTaskID.Int64) + } + if pipeline.IndexingTaskID.Valid { + taskIDs = append(taskIDs, pipeline.IndexingTaskID.Int64) + } + } + + // Check if any tasks are still running + if len(taskIDs) > 0 { + var runningTasks int + err = tx.QueryRow(`SELECT COUNT(*) FROM harmony_task WHERE id = ANY($1)`, taskIDs).Scan(&runningTasks) + if err != nil { + return false, err + } + if runningTasks > 0 { + return false, fmt.Errorf("cannot remove deal pipeline %s: tasks are still running", id) + } + } + + //Mark failure for deal + _, err = tx.Exec(`UPDATE market_mk20_deal SET error = $1 WHERE id = $2`, "Deal pipeline removed by SP", id) + if err != nil { + return false, xerrors.Errorf("failed to mark deal %s as failed", id) + } + + // Remove market_mk20_pipeline entry + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline WHERE id = $1`, id) + if err != nil { + return false, err + } + + // If sector is null, remove related pieceref + for _, pipeline := range pipelines { + if !pipeline.Sector.Valid { + const prefix = "pieceref:" + if strings.HasPrefix(pipeline.Url, prefix) { + refIDStr := pipeline.Url[len(prefix):] + refID, err := strconv.ParseInt(refIDStr, 10, 64) + if err != nil { + return false, fmt.Errorf("invalid refID in URL: %v", err) + } + // Remove from parked_piece_refs where ref_id = refID + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + return false, err + } + } + } + } + return true, nil + }, harmonydb.OptionRetry()) + return err +} + +func (a *WebRPC) mk12DealPipelineRemove(ctx context.Context, uuid string) error { _, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { // First, get deal_pipeline.url, task_ids, and sector values var ( @@ -1176,7 +1275,7 @@ func (a *WebRPC) MK12DealPipelineRemove(ctx context.Context, uuid string) error &url, §or, &commpTaskID, &psdTaskID, &findDealTaskID, &indexingTaskID, ) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { return false, fmt.Errorf("no deal pipeline found with uuid %s", uuid) } return false, err @@ -1253,7 +1352,7 @@ func (a *WebRPC) MK12DealPipelineRemove(ctx context.Context, uuid string) error return err } -type PipelineFailedStats struct { +type MK12PipelineFailedStats struct { DownloadingFailed int64 CommPFailed int64 PSDFailed int64 @@ -1261,7 +1360,7 @@ type PipelineFailedStats struct { IndexFailed int64 } -func (a *WebRPC) MK12PipelineFailedTasks(ctx context.Context) (*PipelineFailedStats, error) { +func (a *WebRPC) MK12PipelineFailedTasks(ctx context.Context) (*MK12PipelineFailedStats, error) { // We'll create a similar query, but this time we coalesce the task IDs from harmony_task. // If the join fails (no matching harmony_task), all joined fields for that task will be NULL. // We detect failure by checking that xxx_task_id IS NOT NULL, after_xxx = false, and that no task record was found in harmony_task. @@ -1280,7 +1379,7 @@ WITH pipeline_data AS ( dp.after_find_deal, pp.task_id AS downloading_task_id FROM market_mk12_deal_pipeline dp - LEFT JOIN parked_pieces pp ON pp.piece_cid = dp.piece_cid + LEFT JOIN parked_pieces pp ON pp.piece_cid = dp.piece_cid AND pp.piece_padded_size = dp.piece_size WHERE dp.complete = false ), tasks AS ( @@ -1359,7 +1458,7 @@ FROM tasks counts := c[0] - return &PipelineFailedStats{ + return &MK12PipelineFailedStats{ DownloadingFailed: counts.DownloadingFailed, CommPFailed: counts.CommPFailed, PSDFailed: counts.PSDFailed, @@ -1368,7 +1467,7 @@ FROM tasks }, nil } -func (a *WebRPC) BulkRestartFailedMarketTasks(ctx context.Context, taskType string) error { +func (a *WebRPC) MK12BulkRestartFailedMarketTasks(ctx context.Context, taskType string) error { didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { var rows *harmonydb.Query var err error @@ -1494,7 +1593,7 @@ func (a *WebRPC) BulkRestartFailedMarketTasks(ctx context.Context, taskType stri return nil } -func (a *WebRPC) BulkRemoveFailedMarketPipelines(ctx context.Context, taskType string) error { +func (a *WebRPC) MK12BulkRemoveFailedMarketPipelines(ctx context.Context, taskType string) error { didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { var rows *harmonydb.Query var err error diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index 8b64ce88b..768a328b8 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -3,15 +3,21 @@ package webrpc import ( "context" "database/sql" + "errors" "fmt" + "strconv" + "strings" + "time" "github.com/ipfs/go-cid" "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/mk20" ) @@ -114,3 +120,444 @@ func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) } return mk20Summaries, nil } + +func (a *WebRPC) MK20DealPipelines(ctx context.Context, limit int, offset int) ([]*MK20DealPipeline, error) { + if limit <= 0 { + limit = 25 + } + if limit > 100 { + limit = 100 + } + if offset < 0 { + offset = 0 + } + + var pipelines []*MK20DealPipeline + err := a.deps.DB.Select(ctx, &pipelines, ` + SELECT + created_at, + id, + sp_id, + contract, + client, + piece_cid, + piece_size, + raw_size, + offline, + url, + indexing, + announce, + allocation_id, + piece_aggregation, + started, + downloaded, + commp_task_id, + after_commp, + deal_aggregation, + aggr_index, + agg_task_id, + aggregated, + sector, + reg_seal_proof, + sector_offset, + sealed, + indexing_created_at, + indexing_task_id, + indexed, + complete + FROM market_mk20_pipeline + ORDER BY created_at DESC + LIMIT $1 OFFSET $2`, limit, offset) + if err != nil { + return nil, fmt.Errorf("failed to fetch deal pipelines: %w", err) + } + + for _, s := range pipelines { + addr, err := address.NewIDAddress(uint64(s.SpId)) + if err != nil { + return nil, xerrors.Errorf("failed to parse the miner ID: %w", err) + } + s.Miner = addr.String() + pcid, err := cid.Parse(s.PieceCid) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(s.PieceSize), + }) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + s.PieceCidV2 = commp.PCidV2().String() + } + + return pipelines, nil +} + +type MK20PipelineFailedStats struct { + DownloadingFailed int64 + CommPFailed int64 + AggFailed int64 + IndexFailed int64 +} + +func (a *WebRPC) MK20PipelineFailedTasks(ctx context.Context) (*MK20PipelineFailedStats, error) { + // We'll create a similar query, but this time we coalesce the task IDs from harmony_task. + // If the join fails (no matching harmony_task), all joined fields for that task will be NULL. + // We detect failure by checking that xxx_task_id IS NOT NULL, after_xxx = false, and that no task record was found in harmony_task. + + const query = ` + WITH pipeline_data AS ( + SELECT dp.id, + dp.complete, + dp.commp_task_id, + dp.agg_task_id, + dp.indexing_task_id, + dp.sector, + dp.after_commp, + dp.aggregated, + pp.task_id AS downloading_task_id + FROM market_mk20_pipeline dp + LEFT JOIN parked_pieces pp ON pp.piece_cid = dp.piece_cid AND pp.piece_padded_size = dp.piece_size + WHERE dp.complete = false + ), + tasks AS ( + SELECT p.*, + dt.id AS downloading_tid, + ct.id AS commp_tid, + pt.id AS agg_tid, + it.id AS index_tid + FROM pipeline_data p + LEFT JOIN harmony_task dt ON dt.id = p.downloading_task_id + LEFT JOIN harmony_task ct ON ct.id = p.commp_task_id + LEFT JOIN harmony_task pt ON pt.id = p.agg_task_id + LEFT JOIN harmony_task it ON it.id = p.indexing_task_id + ) + SELECT + -- Downloading failed: + -- downloading_task_id IS NOT NULL, after_commp = false (haven't completed commp stage), + -- and downloading_tid IS NULL (no harmony_task record) + COUNT(*) FILTER ( + WHERE downloading_task_id IS NOT NULL + AND after_commp = false + AND downloading_tid IS NULL + ) AS downloading_failed, + + -- CommP (verify) failed: + -- commp_task_id IS NOT NULL, after_commp = false, commp_tid IS NULL + COUNT(*) FILTER ( + WHERE commp_task_id IS NOT NULL + AND after_commp = false + AND commp_tid IS NULL + ) AS commp_failed, + + -- Aggregation failed: + -- agg_task_id IS NOT NULL, aggregated = false, agg_tid IS NULL + COUNT(*) FILTER ( + WHERE agg_task_id IS NOT NULL + AND aggregated = false + AND agg_tid IS NULL + ) AS agg_failed, + + -- Index failed: + -- indexing_task_id IS NOT NULL and if we assume indexing is after find_deal: + -- If indexing_task_id is set, we are presumably at indexing stage. + -- If index_tid IS NULL (no task found), then it's failed. + -- We don't have after_index, now at indexing. + COUNT(*) FILTER ( + WHERE indexing_task_id IS NOT NULL + AND index_tid IS NULL + AND aggregated = true + ) AS index_failed + FROM tasks + ` + + var c []struct { + DownloadingFailed int64 `db:"downloading_failed"` + CommPFailed int64 `db:"commp_failed"` + AggFailed int64 `db:"agg_failed"` + IndexFailed int64 `db:"index_failed"` + } + + err := a.deps.DB.Select(ctx, &c, query) + if err != nil { + return nil, xerrors.Errorf("failed to run failed task query: %w", err) + } + + counts := c[0] + + return &MK20PipelineFailedStats{ + DownloadingFailed: counts.DownloadingFailed, + CommPFailed: counts.CommPFailed, + AggFailed: counts.AggFailed, + IndexFailed: counts.IndexFailed, + }, nil +} + +func (a *WebRPC) MK20BulkRestartFailedMarketTasks(ctx context.Context, taskType string) error { + didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + var rows *harmonydb.Query + var err error + + switch taskType { + case "downloading": + rows, err = tx.Query(` + SELECT pp.task_id + FROM market_mk20_pipeline dp + LEFT JOIN parked_pieces pp ON pp.piece_cid = dp.piece_cid AND pp.piece_padded_size = dp.piece_size + LEFT JOIN harmony_task h ON h.id = pp.task_id + WHERE dp.downloaded = false + AND h.id IS NULL + `) + case "commp": + rows, err = tx.Query(` + SELECT dp.commp_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.commp_task_id + WHERE dp.complete = false + AND dp.downloaded = true + AND dp.commp_task_id IS NOT NULL + AND dp.after_commp = false + AND h.id IS NULL + `) + case "aggregate": + rows, err = tx.Query(` + SELECT dp.agg_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND dp.after_commp = true + AND dp.agg_task_id IS NOT NULL + AND dp.aggregated = false + AND h.id IS NULL + `) + case "index": + rows, err = tx.Query(` + SELECT dp.indexing_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.indexing_task_id + WHERE dp.complete = false + AND dp.indexing_task_id IS NOT NULL + AND dp.sealed = true + AND h.id IS NULL + `) + default: + return false, fmt.Errorf("unknown task type: %s", taskType) + } + + if err != nil { + return false, fmt.Errorf("failed to query failed tasks: %w", err) + } + defer rows.Close() + + var taskIDs []int64 + for rows.Next() { + var tid int64 + if err := rows.Scan(&tid); err != nil { + return false, fmt.Errorf("failed to scan task_id: %w", err) + } + taskIDs = append(taskIDs, tid) + } + + if err := rows.Err(); err != nil { + return false, fmt.Errorf("row iteration error: %w", err) + } + + for _, taskID := range taskIDs { + var name string + var posted time.Time + var result bool + err = tx.QueryRow(` + SELECT name, posted, result + FROM harmony_task_history + WHERE task_id = $1 + ORDER BY id DESC LIMIT 1 + `, taskID).Scan(&name, &posted, &result) + if errors.Is(err, pgx.ErrNoRows) { + // No history means can't restart this task + continue + } else if err != nil { + return false, fmt.Errorf("failed to query history: %w", err) + } + + // If result=true means the task ended successfully, no restart needed + if result { + continue + } + + log.Infow("restarting task", "task_id", taskID, "name", name) + + _, err = tx.Exec(` + INSERT INTO harmony_task (id, initiated_by, update_time, posted_time, owner_id, added_by, previous_task, name) + VALUES ($1, NULL, NOW(), $2, NULL, $3, NULL, $4) + `, taskID, posted, a.deps.MachineID, name) + if err != nil { + return false, fmt.Errorf("failed to insert harmony_task for task_id %d: %w", taskID, err) + } + } + + // All done successfully, commit the transaction + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return err + } + if !didCommit { + return fmt.Errorf("transaction did not commit") + } + + return nil +} + +func (a *WebRPC) MK20BulkRemoveFailedMarketPipelines(ctx context.Context, taskType string) error { + didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + var rows *harmonydb.Query + var err error + + // We'll select pipeline fields directly based on the stage conditions + switch taskType { + case "downloading": + rows, err = tx.Query(` + SELECT dp.id, dp.url, dp.sector, + dp.commp_task_id, dp.agg_task_id, dp.indexing_task_id + FROM market_mk20_pipeline dp + LEFT JOIN parked_pieces pp ON pp.piece_cid = dp.piece_cid AND pp.piece_padded_size = dp.piece_size + LEFT JOIN harmony_task h ON h.id = pp.task_id + WHERE dp.complete = false + AND dp.downloaded = false + AND pp.task_id IS NOT NULL + AND h.id IS NULL + `) + case "commp": + rows, err = tx.Query(` + SELECT dp.id, dp.url, dp.sector, + dp.commp_task_id, dp.agg_task_id, dp.indexing_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.commp_task_id + WHERE dp.complete = false + AND dp.downloaded = true + AND dp.commp_task_id IS NOT NULL + AND dp.after_commp = false + AND h.id IS NULL + `) + case "aggregate": + rows, err = tx.Query(` + SELECT dp.id, dp.url, dp.sector, + dp.commp_task_id, dp.agg_task_id, dp.indexing_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND after_commp = true + AND dp.agg_task_id IS NOT NULL + AND dp.aggregated = false + AND h.id IS NULL + `) + case "index": + rows, err = tx.Query(` + SELECT dp.id, dp.url, dp.sector, + dp.commp_task_id, dp.agg_task_id, dp.indexing_task_id + FROM market_mk20_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.indexing_task_id + WHERE dp.complete = false + AND sealed = true + AND dp.indexing_task_id IS NOT NULL + AND h.id IS NULL + `) + default: + return false, fmt.Errorf("unknown task type: %s", taskType) + } + + if err != nil { + return false, fmt.Errorf("failed to query failed pipelines: %w", err) + } + defer rows.Close() + + type pipelineInfo struct { + id string + url string + sector sql.NullInt64 + commpTaskID sql.NullInt64 + aggTaskID sql.NullInt64 + indexingTaskID sql.NullInt64 + } + + var pipelines []pipelineInfo + for rows.Next() { + var p pipelineInfo + if err := rows.Scan(&p.id, &p.url, &p.sector, &p.commpTaskID, &p.aggTaskID, &p.indexingTaskID); err != nil { + return false, fmt.Errorf("failed to scan pipeline info: %w", err) + } + pipelines = append(pipelines, p) + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("row iteration error: %w", err) + } + + for _, p := range pipelines { + // Gather task IDs + var taskIDs []int64 + if p.commpTaskID.Valid { + taskIDs = append(taskIDs, p.commpTaskID.Int64) + } + if p.aggTaskID.Valid { + taskIDs = append(taskIDs, p.aggTaskID.Int64) + } + if p.indexingTaskID.Valid { + taskIDs = append(taskIDs, p.indexingTaskID.Int64) + } + + if len(taskIDs) > 0 { + var runningTasks int + err = tx.QueryRow(`SELECT COUNT(*) FROM harmony_task WHERE id = ANY($1)`, taskIDs).Scan(&runningTasks) + if err != nil { + return false, err + } + if runningTasks > 0 { + // This should not happen if they are failed, but just in case + return false, fmt.Errorf("cannot remove deal pipeline %s: tasks are still running", p.id) + } + } + + _, err = tx.Exec(`UPDATE market_mk20_deal SET error = $1 WHERE id = $2`, "Deal pipeline removed by SP", p.id) + if err != nil { + return false, xerrors.Errorf("store deal failure: updating deal pipeline: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline WHERE id = $1`, p.id) + if err != nil { + return false, err + } + + // If sector is null, remove related pieceref + if !p.sector.Valid { + const prefix = "pieceref:" + if strings.HasPrefix(p.url, prefix) { + refIDStr := p.url[len(prefix):] + refID, err := strconv.ParseInt(refIDStr, 10, 64) + if err != nil { + return false, fmt.Errorf("invalid refID in URL for pipeline %s: %v", p.id, err) + } + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + return false, fmt.Errorf("failed to remove parked_piece_refs for pipeline %s: %w", p.id, err) + } + } + } + + log.Infow("removed failed pipeline", "id", p.id) + } + + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return err + } + if !didCommit { + return fmt.Errorf("transaction did not commit") + } + + return nil +} diff --git a/web/static/pages/mk12-deals/deal-pipelines.mjs b/web/static/pages/mk12-deals/deal-pipelines.mjs index df1a135c2..9e73ac5b2 100644 --- a/web/static/pages/mk12-deals/deal-pipelines.mjs +++ b/web/static/pages/mk12-deals/deal-pipelines.mjs @@ -135,7 +135,7 @@ class DealPipelines extends LitElement { this.requestUpdate(); try { - await RPCCall('BulkRestartFailedMarketTasks', [type]); + await RPCCall('MK12BulkRestartFailedMarketTasks', [type]); await this.loadData(); } catch (err) { console.error('Failed to restart tasks:', err); @@ -152,7 +152,7 @@ class DealPipelines extends LitElement { this.requestUpdate(); try { - await RPCCall('BulkRemoveFailedMarketPipelines', [type]); + await RPCCall('MK12BulkRemoveFailedMarketPipelines', [type]); await this.loadData(); } catch (err) { console.error('Failed to remove pipelines:', err); diff --git a/web/static/pages/mk12-deals/index.html b/web/static/pages/mk12-deals/index.html index dd673b3e2..0964ecce0 100644 --- a/web/static/pages/mk12-deals/index.html +++ b/web/static/pages/mk12-deals/index.html @@ -1,7 +1,7 @@ - Storage Marker + MK12 Storage Deals diff --git a/web/static/pages/mk12-deals/mk12-deals.mjs b/web/static/pages/mk12-deals/mk12-deals.mjs index c9c5a1c1b..056cf0be1 100644 --- a/web/static/pages/mk12-deals/mk12-deals.mjs +++ b/web/static/pages/mk12-deals/mk12-deals.mjs @@ -92,7 +92,7 @@ class MK12DealList extends LitElement {     		${formatDate(deal.created_at)} ${deal.id} ${deal.miner}${deal.piece_cid}${this.formatPieceCid(deal.piece_cid)} ${this.formatBytes(deal.piece_size)}
    +
    Identifier${identifier}
    Error
    PieceCID${data?.piece_cid['/']}
    PieceSize${data?.piece_size}
    Error
    - + +

    Piece Format

    ${this.renderPieceFormat(data?.format)} - ${data?.source_http ? this.renderSourceHTTP(data.source_http) : ''} - ${data?.source_aggregate ? this.renderSourceAggregate(data.source_aggregate) : ''} - ${data?.source_offline ? this.renderSourceOffline(data.source_offline) : ''} - ${data?.source_httpput ? this.renderSourceHttpPut(data.source_httpput) : ''} + +

    Data Source

    + + + + + + + ${this.renderDataSource(data)} + +
    NameDetails
    ${products?.ddo_v1 ? this.renderDDOV1(products.ddo_v1) : ''} - `; } + renderDataSource(data){ + if (!data) return ''; + if (data.source_http) { + return html` +
    HTTP${data?.source_http ? this.renderSourceHTTP(data.source_http) : ''}
    Aggregate${data?.source_aggregate ? this.renderSourceAggregate(data.source_aggregate) : ''}
    Offline${data?.source_offline ? this.renderSourceOffline(data.source_offline) : ''}
    HTTP Put${data?.source_httpput ? this.renderSourceHttpPut(data.source_httpput) : ''}
    - ${format.car ? html`` : ''} - ${format.aggregate - ? html` - - + + + + + + ${format.car ? html`` : ''} + ${format.aggregate + ? html` + ` - : ''} - ${format.raw ? html`` : ''} + : ''} + ${format.raw ? html`` : ''} +
    CarYes
    Aggregate Type${format.aggregate.type}
    ${this.renderAggregateSubs(format.aggregate.sub)}
    Format NameDetails
    Car
    AggregateType ${format.aggregate.type}
    RawYes
    Raw
    `; } @@ -77,7 +123,6 @@ class DealDetails extends LitElement { renderAggregateSubs(subs) { if (!subs?.length) return ''; return html` -
    Aggregate Sub Formats
    @@ -96,39 +141,58 @@ class DealDetails extends LitElement { renderSourceHTTP(src) { return html` -
    Source HTTP
    #CarRawAggregate
    - - - - + + ${src.urls ? this.renderUrls(src.urls) : ''}
    Raw Size${src.rawsize}
    URLs - - - - ${src.urls.map(u => html` - - - - - - `)} - -
    URLPriorityFallback
    ${u.url}${u.priority}${u.fallback ? 'Yes' : 'No'}
    -
    Raw Size${src.rawsize}
    `; } + renderUrls(urls) { + if (!urls?.length) return ''; + return html` + + + + + + + + + ${urls.map(u => html` + + + ${u.priority} + + + `)} + +
    URLHeadersPriorityFallback
    ${u.url} +
    + [SHOW] + 
    ${JSON.stringify(u.headers, null, 2)}
    +
    +     		${u.fallback}
    + ` + } + renderSourceAggregate(src) { return html` -
    Source Aggregate
    ${src.pieces.map((piece, i) => html`
    - Piece ${i + 1} - - - -
    PieceCID${piece.piece_cid['/']}
    Size${piece.size}
    + Piece ${i + 1}: ${piece.piece_cid['/']} +
    + [DETAILS] + 
    +                  
+                      
+                      
+                      
+                      
+                  
    PieceCID${piece.piece_cid['/']}
    Size${piece.piece_size}
    Format${this.renderPieceFormat(piece.format)}
    Source${this.renderDataSource(piece)}
    
+
    +
    `)} `; @@ -136,7 +200,6 @@ class DealDetails extends LitElement { renderSourceOffline(src) { return html` -
    Source Offline
    Raw Size${src.raw_size}
    @@ -145,7 +208,6 @@ class DealDetails extends LitElement { renderSourceHttpPut(src) { return html` -
    Source HTTP PUT
    Raw Size${src.raw_size}
    @@ -157,8 +219,8 @@ class DealDetails extends LitElement {
    DDO v1
    - - + + ${ddo.allocation_id ? html`` : ''} @@ -172,92 +234,3 @@ class DealDetails extends LitElement { } customElements.define('deal-details', DealDetails); -// import { LitElement, html, css } from 'lit'; -// import { customElement, property } from 'lit/decorators.js'; -// -// @customElement('deal-view') -// export class DealView extends LitElement { -// @property({ type: Object }) deal; -// -// static styles = css` -// table { -// border-collapse: collapse; -// width: 100%; -// margin-bottom: 1rem; -// } -// th, td { -// border: 1px solid #ddd; -// padding: 0.5rem; -// vertical-align: top; -// } -// th { -// background-color: #f8f9fa; -// text-align: left; -// } -// .nested-table { -// margin-left: 1rem; -// width: auto; -// } -// `; -// -// renderNested(title, obj) { -// if (!obj) return html``; -// return html` -// -// -// -// ${Object.entries(obj).map(([key, value]) => html` -// -// -// -// -// `)} -// `; -// } -// -// renderRows(data) { -// return Object.entries(data).map(([key, value]) => { -// if (typeof value === 'object' && value !== null && !Array.isArray(value)) { -// return html`${this.renderNested(key, value)}`; -// } else { -// return html` -// -// -// -// -// `; -// } -// }); -// } -// -// render() { -// if (!this.deal) return html`

    No deal provided.

    `; -// return html` -//
    Provider${ddo.provider}
    Client${ddo.client}
    Piece Manager${ddo.piece_manager}
    Client
    Piece Manager
    Duration${ddo.duration}
    Allocation ID${ddo.allocation_id}
    Contract${ddo.contract_address}
    ${title}
    ${key} -// ${typeof value === 'object' && value !== null -// ? html`${this.renderRows(value)}
    ` -// : String(value)} -//
    ${key}${Array.isArray(value) ? html`
    ${JSON.stringify(value, null, 2)}
    ` : String(value)}
    -// -// -// -// -// -// ${this.deal.data ? html` -// -// -// -// ${this.renderNested('data', this.deal.data)} -// ` : null} -// ${this.deal.products?.ddo_v1 ? html` -// -// -// -// ${this.renderNested('DDOV1', this.deal.products.ddo_v1)} -// ` : null} -// -//
    Deal
    Identifier${this.deal.identifier}
    data
    DDOV1
    -// `; -// } -// } - diff --git a/web/static/pages/mk20-deal/index.html b/web/static/pages/mk20-deal/index.html index b29fb114d..f05a776bf 100644 --- a/web/static/pages/mk20-deal/index.html +++ b/web/static/pages/mk20-deal/index.html @@ -21,6 +21,13 @@

    MK20 Deal Info

    +
    +
    +
    + +
    +
    +
    diff --git a/web/static/pages/mk20/ddo-pipeline.mjs b/web/static/pages/mk20/ddo-pipeline.mjs new file mode 100644 index 000000000..e422ebb90 --- /dev/null +++ b/web/static/pages/mk20/ddo-pipeline.mjs @@ -0,0 +1,376 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; +import { formatDate } from '/lib/dateutil.mjs'; + +class MK20DealPipelines extends LitElement { + static properties = { + deals: { type: Array }, + limit: { type: Number }, + offset: { type: Number }, + totalCount: { type: Number }, + failedTasks: { type: Object }, + restartingTaskType: { type: String }, + removingTaskType: { type: String } + }; + + constructor() { + super(); + this.deals = []; + this.limit = 25; + this.offset = 0; + this.totalCount = 0; + this.failedTasks = {}; + this.restartingTaskType = ''; + this.removingTaskType = ''; + this.loadData(); + } + + connectedCallback() { + super.connectedCallback(); + // Set up an interval to update data every 5 seconds + this.intervalId = setInterval(() => this.loadData(), 5000); + } + + disconnectedCallback() { + super.disconnectedCallback(); + // Clear the interval when the element is disconnected + clearInterval(this.intervalId); + } + + async loadData() { + try { + const params = [this.limit, this.offset]; + const deals = await RPCCall('MK20DealPipelines', params); + this.deals = deals; + + // Load failed tasks data + const failed = await RPCCall('MK20PipelineFailedTasks', []); + this.failedTasks = failed || {}; + + this.requestUpdate(); + } catch (error) { + console.error('Failed to load deal pipelines or failed tasks:', error); + } + } + + nextPage() { + this.offset += this.limit; + this.loadData(); + } + + prevPage() { + if (this.offset >= this.limit) { + this.offset -= this.limit; + } else { + this.offset = 0; + } + this.loadData(); + } + + renderFailedTasks() { + const { DownloadingFailed, CommPFailed, AggFailed, IndexFailed } = this.failedTasks; + const entries = []; + + const renderLine = (label, count, type) => { + const isRestarting = this.restartingTaskType === type; + const isRemoving = this.removingTaskType === type; + const isWorking = isRestarting || isRemoving; + return html` +
    + ${label} Task: ${count} +
    + + ${isWorking ? 'Working...' : 'Actions'} + + + +
    +
    + `; + }; + + if (DownloadingFailed > 0) { + entries.push(renderLine('Downloading', DownloadingFailed, 'downloading')); + } + if (CommPFailed > 0) { + entries.push(renderLine('CommP', CommPFailed, 'commp')); + } + if (AggFailed > 0) { + entries.push(renderLine('Aggregate', AggFailed, 'aggregate')); + } + if (IndexFailed > 0) { + entries.push(renderLine('Index', IndexFailed, 'index')); + } + + if (entries.length === 0) { + return null; + } + + return html` +
    +

    Failed Tasks

    + ${entries} +
    + `; + } + + async restartFailedTasks(type) { + this.restartingTaskType = type; + this.removingTaskType = ''; + this.requestUpdate(); + + try { + await RPCCall('MK20BulkRestartFailedMarketTasks', [type]); + await this.loadData(); + } catch (err) { + console.error('Failed to restart tasks:', err); + alert(`Failed to restart ${type} tasks: ${err.message || err}`); + } finally { + this.restartingTaskType = ''; + this.requestUpdate(); + } + } + + async removeFailedPipelines(type) { + this.removingTaskType = type; + this.restartingTaskType = ''; + this.requestUpdate(); + + try { + await RPCCall('MK20BulkRemoveFailedMarketPipelines', [type]); + await this.loadData(); + } catch (err) { + console.error('Failed to remove pipelines:', err); + alert(`Failed to remove ${type} pipelines: ${err.message || err}`); + } finally { + this.removingTaskType = ''; + this.requestUpdate(); + } + } + + render() { + return html` + + + +
    + ${this.renderFailedTasks()} +

    + Deal Pipelines + +

    + + + + + + + + + + + + + ${this.deals.map( + (deal) => html` + + + + + + + + + ` + )} + +
    Created AtUUIDSP IDPiece CIDPiece SizeStatus
    ${formatDate(deal.created_at)} + ${deal.id} + ${deal.miner} + ${this.formatPieceCid(deal.piece_cid)} + ${this.formatBytes(deal.piece_size)}${this.getDealStatus(deal)}
    +
    + + Page ${(this.offset / this.limit) + 1} + +
    +
    + `; + } + + formatPieceCid(pieceCid) { + if (!pieceCid) return ''; + if (pieceCid.length <= 24) { + return pieceCid; + } + const start = pieceCid.substring(0, 16); + const end = pieceCid.substring(pieceCid.length - 8); + return `${start}...${end}`; + } + + formatBytes(bytes) { + const units = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + let size = bytes; + while (size >= 1024 && i < units.length - 1) { + size /= 1024; + i++; + } + if (i === 0) { + return `${size} ${units[i]}`; + } else { + return `${size.toFixed(2)} ${units[i]}`; + } + } + + getDealStatus(deal) { + if (deal.complete) { + return '(#########) Complete'; + } else if (!deal.complete && deal.announce && deal.indexed) { + return '(########.) Announcing'; + } else if (deal.sealed && !deal.indexed) { + return '(#######..) Indexing'; + } else if (deal.sector?.Valid && !deal.sealed) { + return '(######...) Sealing'; + } else if (deal.aggregated && !deal.sector?.Valid) { + return '(#####....) Assigning Sector'; + } else if (deal.after_commp && !deal.aggregated) { + return '(####.....) Aggregating Deal'; + } else if (deal.downloaded && !deal.after_commp) { + return '(###......) CommP'; + } else if (deal.started && !deal.downloaded) { + return '(##.......) Downloading'; + } else { + return '(#........) Accepted'; + } + } + + static styles = css` + .pagination-controls { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + + .info-btn { + position: relative; + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + font-size: 1em; + margin-left: 8px; + } + + .tooltip-text { + display: none; + position: absolute; + top: 50%; + left: 120%; + transform: translateY(-50%); + min-width: 440px; + max-width: 600px; + background-color: #333; + color: #fff; + padding: 8px; + border-radius: 4px; + font-size: 0.8em; + text-align: left; + white-space: normal; + z-index: 10; + } + + .info-btn:hover .tooltip-text { + display: block; + } + + .copy-btn { + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + padding: 0 0 0 5px; + } + + .copy-btn svg { + vertical-align: middle; + } + + .copy-btn:hover { + color: #0d6efd; + } + + .failed-tasks { + margin-bottom: 1rem; + } + .failed-tasks h2 { + margin: 0 0 0.5rem 0; + } + + details > summary { + display: inline-block; + cursor: pointer; + outline: none; + } + + .btn { + margin: 0 4px; + } + `; +} + +customElements.define('mk20-deal-pipelines', MK20DealPipelines); diff --git a/web/static/pages/mk20/ddo.mjs b/web/static/pages/mk20/ddo.mjs index bec04db79..c6c4b39b2 100644 --- a/web/static/pages/mk20/ddo.mjs +++ b/web/static/pages/mk20/ddo.mjs @@ -59,6 +59,7 @@ class MK20DDODealList extends LitElement {
    +

    DDO Deal List - Created At${formatDate(entry.pipeline.created_at)} - Piece CID${entry.pipeline.piece_cid} - Piece Size${this.toHumanBytes(entry.pipeline.piece_size)} - Raw Size${entry.pipeline.raw_size.Valid ? this.toHumanBytes(entry.pipeline.raw_size.Int64) : 'N/A'} - Offline - URL${entry.pipeline.url.Valid ? entry.pipeline.url.String : 'N/A'} - Headers
    ${JSON.stringify(entry.pipeline.headers, null, 2)}
    - Should Index${this.renderNullableYesNo(entry.pipeline.should_index.Bool)} + Created At${formatDate(entry.mk12_pipeline.created_at)} + Piece CID${entry.mk12_pipeline.piece_cid} + Piece Size${this.toHumanBytes(entry.mk12_pipeline.piece_size)} + Raw Size${entry.mk12_pipeline.raw_size.Valid ? this.toHumanBytes(entry.mk12_pipeline.raw_size.Int64) : 'N/A'} + Offline + URL${entry.mk12_pipeline.url.Valid ? entry.mk12_pipeline.url.String : 'N/A'} + Headers
    ${JSON.stringify(entry.mk12_pipeline.headers, null, 2)}
    + Should Index${this.renderNullableYesNo(entry.mk12_pipeline.should_index.Bool)} Announce - ${this.renderNullableYesNo(entry.pipeline.announce.Bool)} + ${this.renderNullableYesNo(entry.mk12_pipeline.announce.Bool)}

    Progress šŸ› ļø
    Data Fetched - ${this.renderNullableDoneNotDone(entry.pipeline.started.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.started.Bool)} After Commp - ${this.renderNullableDoneNotDone(entry.pipeline.after_commp.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.after_commp.Bool)} After PSD - ${this.renderNullableDoneNotDone(entry.pipeline.after_psd.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.after_psd.Bool)} After Find Deal - ${this.renderNullableDoneNotDone(entry.pipeline.after_find_deal.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.after_find_deal.Bool)} Sealed - ${this.renderNullableDoneNotDone(entry.pipeline.sealed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.sealed.Bool)} Indexed - ${this.renderNullableDoneNotDone(entry.pipeline.indexed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk12_pipeline.indexed.Bool)} Announced - +
    Early States šŸŒæ
    Commp Task ID - ${entry.pipeline.commp_task_id.Valid - ? html`` + ${entry.mk12_pipeline.commp_task_id.Valid + ? html`` : 'N/A'} PSD Task ID - ${entry.pipeline.psd_task_id.Valid - ? html`` + ${entry.mk12_pipeline.psd_task_id.Valid + ? html`` : 'N/A'} - PSD Wait Time${entry.pipeline.psd_wait_time.Valid ? formatDate(entry.pipeline.psd_wait_time.Time) : 'N/A'} + PSD Wait Time${entry.mk12_pipeline.psd_wait_time.Valid ? formatDate(entry.mk12_pipeline.psd_wait_time.Time) : 'N/A'} Find Deal Task ID - ${entry.pipeline.find_deal_task_id.Valid - ? html`` + ${entry.mk12_pipeline.find_deal_task_id.Valid + ? html`` : 'N/A'}
    Sealing šŸ“¦
    - Sector${entry.pipeline.sector.Valid ? html`${entry.pipeline.sector.Int64}` : 'N/A'} - Reg Seal Proof${entry.pipeline.reg_seal_proof.Valid ? entry.pipeline.reg_seal_proof.Int64 : 'N/A'} - Sector Offset${entry.pipeline.sector_offset.Valid ? entry.pipeline.sector_offset.Int64 : 'N/A'} + Sector${entry.mk12_pipeline.sector.Valid ? html`${entry.mk12_pipeline.sector.Int64}` : 'N/A'} + Reg Seal Proof${entry.mk12_pipeline.reg_seal_proof.Valid ? entry.pipeline.reg_seal_proof.Int64 : 'N/A'} + Sector Offset${entry.mk12_pipeline.sector_offset.Valid ? entry.pipeline.sector_offset.Int64 : 'N/A'}
    Indexing šŸ”
    - Indexing Created At${entry.pipeline.indexing_created_at.Valid ? formatDate(entry.pipeline.indexing_created_at.Time) : 'N/A'} + Indexing Created At${entry.mk12_pipeline.indexing_created_at.Valid ? formatDate(entry.mk12_pipeline.indexing_created_at.Time) : 'N/A'} Indexing Task ID - ${entry.pipeline.indexing_task_id.Valid - ? html`` + ${entry.mk12_pipeline.indexing_task_id.Valid + ? html`` : 'N/A'} @@ -324,7 +324,7 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { `)} ` : ''} - ${this.DealData && this.DealData.mk20.length > 0 ? html` + ${this.DealData?.mk20?.length > 0 ? html`

    Related MK20 Deals

    ${this.DealData.mk20.map((entry) => html`

    Deal ${entry.deal.deal.identifier}

    @@ -333,20 +333,9 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { ID${entry.deal.deal.identifier}
    Deal Data āš™ļø
    - Piece CID${entry.deal.deal.data.piece_cid} + Piece CID${entry.deal.deal.data.piece_cid['/']} Piece Size${this.toHumanBytes(entry.deal.deal.data.piece_size)} -
    Data Source šŸ“„ļø
    - - URL Headers - -
    - [SHOW] - 
    ${JSON.stringify(entry.deal.url_headers, null, 2)}
    -
    - - -
    Status šŸŸ¢ļøšŸ”“
    Error${entry.deal.error.Valid ? entry.deal.error.String : 'N/A'} ${(() => { @@ -371,7 +360,7 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { ${matchingPieceDeals.map((item) => html` - ${item.id} + ${item.id} ${item.boost_deal ? 'Boost' : (item.legacy_deal ? 'Legacy' : 'DDO')} ${item.miner} ${item.chain_deal_id} @@ -387,96 +376,83 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { `; } })()} - ${entry.pipeline ? html` + ${entry.mk20_pipeline ? html`
    PIPELINE ACTIVE
    Controls - + - Created At${formatDate(entry.pipeline.created_at)} - Piece CID${entry.pipeline.piece_cid} - Piece Size${this.toHumanBytes(entry.pipeline.piece_size)} - Raw Size${entry.pipeline.raw_size.Valid ? this.toHumanBytes(entry.pipeline.raw_size.Int64) : 'N/A'} - Offline - URL${entry.pipeline.url.Valid ? entry.pipeline.url.String : 'N/A'} - Headers
    ${JSON.stringify(entry.pipeline.headers, null, 2)}
    - Should Index${this.renderNullableYesNo(entry.pipeline.should_index.Bool)} + Created At${formatDate(entry.mk20_pipeline.created_at)} + Piece CID${entry.mk20_pipeline.piece_cid} + Piece Size${this.toHumanBytes(entry.mk20_pipeline.piece_size)} + Raw Size${entry.mk20_pipeline.raw_size.Valid ? this.toHumanBytes(entry.mk20_pipeline.raw_size.Int64) : 'N/A'} + Offline + URL${entry.mk20_pipeline.url.Valid ? entry.mk20_pipeline.url.String : 'N/A'} + Headers
    ${JSON.stringify(entry.mk20_pipeline.headers, null, 2)}
    + Should Index${this.renderNullableYesNo(entry.mk20_pipeline.indexing.Bool)} Announce - ${this.renderNullableYesNo(entry.pipeline.announce.Bool)} + ${this.renderNullableYesNo(entry.mk20_pipeline.announce.Bool)}
    Progress šŸ› ļø
    Data Fetched - ${this.renderNullableDoneNotDone(entry.pipeline.started.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.started.Bool)} After Commp - ${this.renderNullableDoneNotDone(entry.pipeline.after_commp.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.after_commp.Bool)} - After PSD - ${this.renderNullableDoneNotDone(entry.pipeline.after_psd.Bool)} - - - After Find Deal - ${this.renderNullableDoneNotDone(entry.pipeline.after_find_deal.Bool)} + Aggregated + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.aggregated.Bool)} Sealed - ${this.renderNullableDoneNotDone(entry.pipeline.sealed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.sealed.Bool)} Indexed - ${this.renderNullableDoneNotDone(entry.pipeline.indexed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.indexed.Bool)} Announced - +
    Early States šŸŒæ
    Commp Task ID - ${entry.pipeline.commp_task_id.Valid - ? html`` + ${entry.mk20_pipeline.commp_task_id.Valid + ? html`` : 'N/A'} - PSD Task ID - - ${entry.pipeline.psd_task_id.Valid - ? html`` - : 'N/A'} - - - PSD Wait Time${entry.pipeline.psd_wait_time.Valid ? formatDate(entry.pipeline.psd_wait_time.Time) : 'N/A'} - - Find Deal Task ID + Aggregation Task ID - ${entry.pipeline.find_deal_task_id.Valid - ? html`` + ${entry.mk20_pipeline.agg_task_id.Valid + ? html`` : 'N/A'}
    Sealing šŸ“¦
    - Sector${entry.pipeline.sector.Valid ? html`${entry.pipeline.sector.Int64}` : 'N/A'} - Reg Seal Proof${entry.pipeline.reg_seal_proof.Valid ? entry.pipeline.reg_seal_proof.Int64 : 'N/A'} - Sector Offset${entry.pipeline.sector_offset.Valid ? entry.pipeline.sector_offset.Int64 : 'N/A'} + Sector${entry.mk20_pipeline.sector.Valid ? html`${entry.mk20_pipeline.sector.Int64}` : 'N/A'} + Reg Seal Proof${entry.mk20_pipeline.reg_seal_proof.Valid ? entry.mk20_pipeline.reg_seal_proof.Int64 : 'N/A'} + Sector Offset${entry.mk20_pipeline.sector_offset.Valid ? entry.mk20_pipeline.sector_offset.Int64 : 'N/A'}
    Indexing šŸ”
    - Indexing Created At${entry.pipeline.indexing_created_at.Valid ? formatDate(entry.pipeline.indexing_created_at.Time) : 'N/A'} + Indexing Created At${entry.mk20_pipeline.indexing_created_at.Valid ? formatDate(entry.mk20_pipeline.indexing_created_at.Time) : 'N/A'} Indexing Task ID - ${entry.pipeline.indexing_task_id.Valid - ? html`` + ${entry.mk20_pipeline.indexing_task_id.Valid + ? html`` : 'N/A'} diff --git a/web/static/ux/curio-ux.mjs b/web/static/ux/curio-ux.mjs index d360b2e38..7ed8af5de 100644 --- a/web/static/ux/curio-ux.mjs +++ b/web/static/ux/curio-ux.mjs @@ -205,7 +205,9 @@ class CurioUX extends LitElement {
  • - + + + MK20 From f67021071ccd3a8a585f062624afc7ec4318009f Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 28 May 2025 20:18:08 +0400 Subject: [PATCH 10/55] UI, reindex, retrievals --- cmd/curio/tasks/tasks.go | 4 +- .../harmonydb/sql/20250505-market_mk20.sql | 8 +- lib/cachedreader/cachedreader.go | 81 ++- market/indexstore/indexstore.go | 101 +++- market/indexstore/indexstore_test.go | 16 + market/mk12/http/http.go | 8 +- market/mk20/http/http.go | 14 +- pdp/handlers.go | 22 +- tasks/gc/pipeline_meta_gc.go | 4 +- tasks/indexing/task_check_indexes.go | 554 +++++++++++++----- tasks/indexing/task_indexing.go | 69 ++- tasks/indexing/task_ipni.go | 48 +- web/api/webrpc/market.go | 5 + web/static/pages/actor/actor-detail.mjs | 4 +- web/static/pages/mk20-deal/deal.mjs | 51 +- web/static/pages/piece/piece-info.mjs | 20 +- 16 files changed, 744 insertions(+), 265 deletions(-) diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index e0ffd9045..e8ef56e40 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -297,8 +297,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan idxMax := taskhelp.Max(cfg.Subsystems.IndexingMaxTasks) - indexingTask := indexing.NewIndexingTask(db, sc, iStore, dependencies.CachedPieceReader, cfg, idxMax) - ipniTask := indexing.NewIPNITask(db, sc, iStore, dependencies.CachedPieceReader, cfg, idxMax) + indexingTask := indexing.NewIndexingTask(db, sc, iStore, dependencies.SectorReader, dependencies.CachedPieceReader, cfg, idxMax) + ipniTask := indexing.NewIPNITask(db, sc, iStore, dependencies.SectorReader, dependencies.CachedPieceReader, cfg, idxMax) activeTasks = append(activeTasks, ipniTask, indexingTask) if cfg.HTTP.Enable { diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index eca6f7a9c..3faa3c1c7 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -22,6 +22,9 @@ DROP CONSTRAINT IF EXISTS market_piece_deal_identity_key; ALTER TABLE market_piece_deal ADD PRIMARY KEY (sp_id, id, piece_cid, piece_length); +-- Add a column to relate a piece park piece to mk20 deal +ALTER TABLE market_piece_deal +ADD COLUMN piece_ref BIGINT; -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). @@ -35,6 +38,7 @@ CREATE OR REPLACE FUNCTION process_piece_deal( _piece_length BIGINT, -- padded length _raw_size BIGINT, _indexed BOOLEAN, + _piece_ref BIGINT DEFAULT NULL, _legacy_deal BOOLEAN DEFAULT FALSE, _chain_deal_id BIGINT DEFAULT 0 ) @@ -52,10 +56,10 @@ BEGIN -- Insert into the market_piece_deal table INSERT INTO market_piece_deal ( id, piece_cid, boost_deal, legacy_deal, chain_deal_id, - sp_id, sector_num, piece_offset, piece_length, raw_size + sp_id, sector_num, piece_offset, piece_length, raw_size, piece_ref ) VALUES ( _id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id, - _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size + _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size, _piece_ref ) ON CONFLICT (sp_id, id, piece_cid, piece_length) DO NOTHING; END; diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index b2da0cd52..88976bab1 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -131,9 +131,17 @@ func (r *cachedSectionReader) Close() error { return nil } -func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { // Get all deals containing this piece + commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) + if err != nil { + return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + + pieceCid := commp.PCidV1() + pieceSize := commp.PieceInfo().Size + var deals []struct { SpID abi.ActorID `db:"sp_id"` Sector abi.SectorNumber `db:"sector_num"` @@ -142,7 +150,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec Proof abi.RegisteredSealProof `db:"reg_seal_proof"` } - err := cpr.db.Select(ctx, &deals, `SELECT + err = cpr.db.Select(ctx, &deals, `SELECT mpd.sp_id, mpd.sector_num, mpd.piece_offset, @@ -189,14 +197,22 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec return nil, 0, merr } -func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCid cid.Cid, pieceSize abi.PaddedPieceSize) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { + commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) + if err != nil { + return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + + pieceCid := commp.PCidV1() + pieceSize := commp.PieceInfo().Size + // Query parked_pieces and parked_piece_refs in one go var pieceData []struct { ID int64 `db:"id"` PieceRawSize int64 `db:"piece_raw_size"` } - err := cpr.db.Select(ctx, &pieceData, ` + err = cpr.db.Select(ctx, &pieceData, ` SELECT pp.id, pp.piece_raw_size @@ -222,6 +238,27 @@ func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, p return reader, abi.UnpaddedPieceSize(pieceData[0].PieceRawSize), nil } +type SubPieceReader struct { + sr *io.SectionReader + r io.Closer +} + +func (s SubPieceReader) Read(p []byte) (n int, err error) { + return s.sr.Read(p) +} + +func (s SubPieceReader) Close() error { + return s.r.Close() +} + +func (s SubPieceReader) Seek(offset int64, whence int) (int64, error) { + return s.sr.Seek(offset, whence) +} + +func (s SubPieceReader) ReadAt(p []byte, off int64) (n int, err error) { + return s.sr.ReadAt(p, off) +} + func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { pieces, err := cpr.idxStor.FindPieceInAggregate(ctx, pieceCidV2) if err != nil { @@ -229,27 +266,28 @@ func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, p } if len(pieces) == 0 { - return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) + return nil, 0, fmt.Errorf("subpiece not found in any aggregate piece") } + var merr error + for _, p := range pieces { - commp, err := commcidv2.CommPFromPCidV2(p) - if err != nil { - return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) - } - reader, payloadSize, err := cpr.getPieceReaderFromPiecePark(ctx, commp.PCidV1(), commp.PieceInfo().Size) + reader, _, err := cpr.getPieceReaderFromPiecePark(ctx, p.Cid) if err != nil { - log.Warnw("failed to get piece reader from piece park", "piececid", commp.PCidV1(), "piece size", commp.PieceInfo().Size, "err", err) - reader, payloadSize, err = cpr.getPieceReaderFromSector(ctx, commp.PCidV1(), commp.PieceInfo().Size) + log.Warnw("failed to get piece reader from piece park", "piececid", p.Cid.String(), "err", err) + reader, _, err = cpr.getPieceReaderFromSector(ctx, p.Cid) if err != nil { - log.Errorw("failed to get piece reader from sector", "piececid", commp.PCidV1(), "piece size", commp.PieceInfo().Size, "err", err) + log.Errorw("failed to get piece reader from sector", "piececid", p.Cid.String(), "err", err) + merr = multierror.Append(merr, err) continue } - return reader, payloadSize, nil + sr := io.NewSectionReader(reader, int64(p.Offset), int64(p.Size)) + return SubPieceReader{r: reader, sr: sr}, abi.UnpaddedPieceSize(p.Size), nil } - return reader, payloadSize, nil + sr := io.NewSectionReader(reader, int64(p.Offset), int64(p.Size)) + return SubPieceReader{r: reader, sr: sr}, abi.UnpaddedPieceSize(p.Size), nil } - return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) + return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", merr) } func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { @@ -294,22 +332,21 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid readerCtx, readerCtxCancel := context.WithCancel(context.Background()) defer close(r.ready) - reader, size, err := cpr.getPieceReaderFromAggregate(readerCtx, pieceCid) + reader, size, err := cpr.getPieceReaderFromAggregate(readerCtx, pieceCidV2) if err != nil { - log.Warnw("failed to get piece reader from aggregate", "piececid", pieceCid, "piece size", pieceSize, "err", err) + log.Warnw("failed to get piece reader from aggregate", "piececid", pieceCidV2.String(), "err", err) aerr := err - reader, size, err = cpr.getPieceReaderFromSector(readerCtx, pieceCid, pieceSize) + reader, size, err = cpr.getPieceReaderFromSector(readerCtx, pieceCidV2) if err != nil { - log.Warnw("failed to get piece reader from sector", "piececid", pieceCid, "piece size", pieceSize, "err", err) + log.Warnw("failed to get piece reader from sector", "piececid", pieceCidV2.String(), "err", err) serr := err // Try getPieceReaderFromPiecePark - reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCid, pieceSize) + reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCidV2) if err != nil { log.Errorw("failed to get piece reader from piece park", "piececid", pieceCid, "piece size", pieceSize, "err", err) finalErr := fmt.Errorf("failed to get piece reader from aggregate, sector or piece park: %w, %w, %w", aerr, serr, err) - // Cache the error in the error cache cpr.pieceErrorCacheMu.Lock() _ = cpr.pieceErrorCache.Set(cacheKey, &cachedError{err: finalErr, pieceCid: pieceCid}) diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index 49dd463fd..aebf887d8 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -123,7 +123,7 @@ func (i *IndexStore) Start(ctx context.Context, test bool) error { session.Close() // Recreate session with the keyspace - i.cluster.Keyspace = keyspace + i.cluster.Keyspace = keyspaceName session, err = i.cluster.CreateSession() if err != nil { return xerrors.Errorf("creating cassandra session: %w", err) @@ -477,25 +477,108 @@ func (i *IndexStore) InsertAggregateIndex(ctx context.Context, aggregatePieceCid return nil } -func (i *IndexStore) FindPieceInAggregate(ctx context.Context, pieceCid cid.Cid) ([]cid.Cid, error) { - qry := `SELECT AggregatePieceCid FROM PieceToAggregatePiece WHERE PieceCid = ?` +func (i *IndexStore) FindPieceInAggregate(ctx context.Context, pieceCid cid.Cid) ([]Record, error) { + var recs []Record + qry := `SELECT AggregatePieceCid, UnpaddedOffset, UnpaddedLength FROM PieceToAggregatePiece WHERE PieceCid = ?` iter := i.session.Query(qry, pieceCid.Bytes()).WithContext(ctx).Iter() - var aggregatePieceCidBytes []cid.Cid var r []byte - for iter.Scan(&r) { + var idx, length int64 + for iter.Scan(&r, &idx, &length) { c, err := cid.Cast(r) if err != nil { return nil, xerrors.Errorf("casting aggregate piece cid: %w", err) } - aggregatePieceCidBytes = append(aggregatePieceCidBytes, c) + recs = append(recs, Record{ + Cid: c, + Offset: uint64(idx), + Size: uint64(length), + }) r = make([]byte, 0) } if err := iter.Close(); err != nil { return nil, xerrors.Errorf("iterating aggregate piece cid (P:0x%02x): %w", pieceCid.Bytes(), err) } - if len(aggregatePieceCidBytes) == 0 { - return nil, nil + return recs, nil +} + +func (i *IndexStore) UpdatePieceCidV1ToV2(ctx context.Context, pieceCidV1 cid.Cid, pieceCidV2 cid.Cid) error { + //updateQry := `UPDATE PayloadToPieces SET PieceCid = ? WHERE PieceCid = ?` + //if err := i.session.Query(updateQry, pieceCidV1.Bytes(), pieceCidV2.Bytes()).WithContext(ctx).Exec(); err != nil { + // return xerrors.Errorf("updating piece cid v1 to v2: %w", err) + //} + //return nil + + p1 := pieceCidV1.Bytes() + p2 := pieceCidV2.Bytes() + + // First, select all PayloadMultihash for the given PieceCid from PieceBlockOffsetSize + selectQry := `SELECT PayloadMultihash FROM PieceBlockOffsetSize WHERE PieceCid = ?` + iter := i.session.Query(selectQry, p1).WithContext(ctx).Iter() + + var payloadMultihashBytes []byte + var payloadMultihashes [][]byte + for iter.Scan(&payloadMultihashBytes) { + // Copy the bytes since the slice will be overwritten + mhCopy := make([]byte, len(payloadMultihashBytes)) + copy(mhCopy, payloadMultihashBytes) + payloadMultihashes = append(payloadMultihashes, mhCopy) + } + if err := iter.Close(); err != nil { + return xerrors.Errorf("scanning PayloadMultihash for piece %s: %w", pieceCidV1.String(), err) + } + + // Prepare batch replace for PayloadToPieces + updatePiecesQry := `UPDATE PayloadToPieces SET PieceCid = ? WHERE PayloadMultihash = ? AND PieceCid = ?` + batch := i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batchSize := i.settings.InsertBatchSize + + for idx, payloadMH := range payloadMultihashes { + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: updatePiecesQry, + Args: []interface{}{p2, payloadMH, p1}, + Idempotent: true, + }) + + if len(batch.Entries) >= batchSize || idx == len(payloadMultihashes)-1 { + if err := i.executeBatchWithRetry(ctx, batch, pieceCidV1); err != nil { + return xerrors.Errorf("executing batch replace for PayloadToPieces for piece %s: %w", pieceCidV1, err) + } + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + } + + if len(batch.Entries) >= 0 { + if err := i.executeBatchWithRetry(ctx, batch, pieceCidV1); err != nil { + return xerrors.Errorf("executing batch replace for PayloadToPieces for piece %s: %w", pieceCidV1, err) + } } - return aggregatePieceCidBytes, nil + + // Prepare batch replace for PieceBlockOffsetSize + updatePiecesQry = `UPDATE PieceBlockOffsetSize SET PieceCid = ? WHERE PayloadMultihash = ? AND PieceCid = ?` + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + batchSize = i.settings.InsertBatchSize + + for idx, payloadMH := range payloadMultihashes { + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: updatePiecesQry, + Args: []interface{}{p2, payloadMH, p1}, + Idempotent: true, + }) + + if len(batch.Entries) >= batchSize || idx == len(payloadMultihashes)-1 { + if err := i.executeBatchWithRetry(ctx, batch, pieceCidV1); err != nil { + return xerrors.Errorf("executing batch replace for PieceBlockOffsetSize for piece %s: %w", pieceCidV1, err) + } + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + } + + if len(batch.Entries) >= 0 { + if err := i.executeBatchWithRetry(ctx, batch, pieceCidV1); err != nil { + return xerrors.Errorf("executing batch replace for PieceBlockOffsetSize for piece %s: %w", pieceCidV1, err) + } + } + + return nil } diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index 79ee4fc4d..92caad7e2 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -120,9 +120,25 @@ func TestNewIndexStore(t *testing.T) { err = idxStore.session.Query("SELECT * FROM PieceToAggregatePiece").Exec() require.NoError(t, err) + aggrRec := Record{ + Cid: commp.PieceCID, + Offset: 0, + Size: 100, + } + + err = idxStore.InsertAggregateIndex(ctx, commp.PieceCID, []Record{aggrRec}) + require.NoError(t, err) + + x, err := idxStore.FindPieceInAggregate(ctx, commp.PieceCID) + require.NoError(t, err) + require.Len(t, x, 1) + require.Equal(t, x[0].Cid, commp.PieceCID) + // Drop the tables err = idxStore.session.Query("DROP TABLE PayloadToPieces").Exec() require.NoError(t, err) err = idxStore.session.Query("DROP TABLE PieceBlockOffsetSize").Exec() require.NoError(t, err) + err = idxStore.session.Query("DROP TABLE piecetoaggregatepiece").Exec() + require.NoError(t, err) } diff --git a/market/mk12/http/http.go b/market/mk12/http/http.go index cf478f4c6..f9aa00f75 100644 --- a/market/mk12/http/http.go +++ b/market/mk12/http/http.go @@ -22,6 +22,8 @@ import ( storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) +const requestTimeout = 10 * time.Second + var log = logging.Logger("mk12httphdlr") // Redirector struct with a database connection @@ -49,9 +51,9 @@ func NewMK12DealHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_m func Router(mdh *MK12DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Post("/store", mdh.mk12deal) - mux.Get("/ask", mdh.mk12ask) - mux.Get("/status", mdh.mk12status) + mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk12deal), requestTimeout, "timeout reading request")) + mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk12status), requestTimeout, "timeout reading request")) + mux.Method("GET", "/ask", http.TimeoutHandler(http.HandlerFunc(mdh.mk12ask), requestTimeout, "timeout reading request")) return mux } diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 41abe2811..3c14bbdfa 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -38,6 +38,8 @@ var log = logging.Logger("mk20httphdlr") const maxPutBodySize int64 = 64 << 30 // 64 GiB +const requestTimeout = 10 * time.Second + type MK20DealHandler struct { cfg *config.CurioConfig db *harmonydb.DB // Replace with your actual DB wrapper if different @@ -64,15 +66,11 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { func Router(mdh *MK20DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), 10*time.Second, "timeout reading request")) - mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), 10*time.Second, "timeout reading request")) - mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), 10*time.Second, "timeout reading request")) + mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "timeout reading request")) + mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "timeout reading request")) + mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "timeout reading request")) mux.Put("/data", mdh.mk20UploadDealData) - mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), 10*time.Second, "timeout reading request")) - //mux.Post("/store", mdh.mk20deal) - //mux.Get("/status", mdh.mk20status) - //mux.Get("/contracts", mdh.mk20supportedContracts) - //mux.Get("/info", mdh.info) + mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "timeout reading request")) return mux } diff --git a/pdp/handlers.go b/pdp/handlers.go index a64b47a4d..75d06b250 100644 --- a/pdp/handlers.go +++ b/pdp/handlers.go @@ -33,6 +33,8 @@ import ( types2 "github.com/filecoin-project/lotus/chain/types" ) +const requestTimeout = 10 * time.Second + // PDPService represents the service for managing proof sets and pieces type PDPService struct { db *harmonydb.DB @@ -67,23 +69,23 @@ func Routes(p *PDPService) http.Handler { // Routes for proof sets r.Route("/proof-sets", func(r chi.Router) { // POST /pdp/proof-sets - Create a new proof set - r.Post("/", p.handleCreateProofSet) + r.Method("POST", "/", http.TimeoutHandler(http.HandlerFunc(p.handleCreateProofSet), requestTimeout, "request timeout")) // GET /pdp/proof-sets/created/{txHash} - Get the status of a proof set creation - r.Get("/created/{txHash}", p.handleGetProofSetCreationStatus) + r.Method("GET", "/created/{txHash}", http.TimeoutHandler(http.HandlerFunc(p.handleGetProofSetCreationStatus), requestTimeout, "request timeout")) // Individual proof set routes r.Route("/{proofSetID}", func(r chi.Router) { // GET /pdp/proof-sets/{set-id} - r.Get("/", p.handleGetProofSet) + r.Method("GET", "/", http.TimeoutHandler(http.HandlerFunc(p.handleGetProofSet), requestTimeout, "request timeout")) // DEL /pdp/proof-sets/{set-id} - r.Delete("/", p.handleDeleteProofSet) + r.Method("DELETE", "/", http.TimeoutHandler(http.HandlerFunc(p.handleDeleteProofSet), requestTimeout, "request timeout")) // Routes for roots within a proof set r.Route("/roots", func(r chi.Router) { // POST /pdp/proof-sets/{set-id}/roots - r.Post("/", p.handleAddRootToProofSet) + r.Method("POST", "/", http.TimeoutHandler(http.HandlerFunc(p.handleAddRootToProofSet), requestTimeout, "request timeout")) // GET /pdp/proof-sets/{set-id}/roots/added/{txHash} r.Get("/added/{txHash}", p.handleGetRootAdditionStatus) @@ -91,23 +93,23 @@ func Routes(p *PDPService) http.Handler { // Individual root routes r.Route("/{rootID}", func(r chi.Router) { // GET /pdp/proof-sets/{set-id}/roots/{root-id} - r.Get("/", p.handleGetProofSetRoot) + r.Method("GET", "/", http.TimeoutHandler(http.HandlerFunc(p.handleGetProofSetRoot), requestTimeout, "request timeout")) // DEL /pdp/proof-sets/{set-id}/roots/{root-id} - r.Delete("/", p.handleDeleteProofSetRoot) + r.Method("DELETE", "/", http.TimeoutHandler(http.HandlerFunc(p.handleDeleteProofSetRoot), requestTimeout, "request timeout")) }) }) }) }) - r.Get("/ping", p.handlePing) + r.Method("GET", "/ping", http.TimeoutHandler(http.HandlerFunc(p.handlePing), requestTimeout, "request timeout")) // Routes for piece storage and retrieval // POST /pdp/piece - r.Post("/piece", p.handlePiecePost) + r.Method("POST", "/piece", http.TimeoutHandler(http.HandlerFunc(p.handlePiecePost), requestTimeout, "request timeout")) // GET /pdp/piece/ - r.Get("/piece/", p.handleFindPiece) + r.Method("GET", "/piece", http.TimeoutHandler(http.HandlerFunc(p.handleFindPiece), requestTimeout, "request timeout")) // PUT /pdp/piece/upload/{uploadUUID} r.Put("/piece/upload/{uploadUUID}", p.handlePieceUpload) diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index a28287024..7a36a81a7 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -163,7 +163,7 @@ func (s *PipelineGC) cleanupUpgrade() error { func (s *PipelineGC) cleanupMK12DealPipeline() error { ctx := context.Background() - _, err := s.db.Exec(ctx, `DELETE FROM market_mk12_deal_pipeline WHERE complete = TRUE;`) + _, err := s.db.Exec(ctx, `DELETE FROM market_mk12_deal_pipeline WHERE (should_index = FALSE OR indexed = TRUE) AND complete = TRUE;`) if err != nil { return xerrors.Errorf("failed to clean up sealed deals: %w", err) } @@ -191,7 +191,7 @@ func (s *PipelineGC) cleanupMK20DealPipeline() error { if err != nil { return xerrors.Errorf("failed to clean up download pipeline: %w", err) } - _, err = s.db.Exec(ctx, `DELETE FROM market_mk20_pipeline WHERE complete = TRUE;`) + _, err = s.db.Exec(ctx, `DELETE FROM market_mk20_pipeline WHERE (indexing = FALSE OR indexed = TRUE) AND complete = TRUE;`) if err != nil { return xerrors.Errorf("failed to clean up sealed deals: %w", err) } diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index 178860316..73084c213 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -3,11 +3,17 @@ package indexing import ( "bytes" "context" + "database/sql" + "fmt" + "net/url" "time" + "github.com/google/uuid" "github.com/ipfs/go-cid" + "github.com/oklog/ulid" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -16,6 +22,7 @@ import ( "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/mk20" ) const CheckIndexInterval = 9 * time.Minute @@ -60,21 +67,28 @@ func (c *CheckIndexesTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) return false, xerrors.Errorf("checking IPNI: %w", err) } + err = c.checkIPNIMK20(ctx, taskID) + if err != nil { + return false, xerrors.Errorf("checking IPNI for MK20 deals: %w", err) + } + return true, nil } func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask.TaskID) error { type checkEntry struct { - PieceCid string `db:"piece_cid"` - PieceLen int64 `db:"piece_length"` - PieceOff int64 `db:"piece_offset"` - SPID int64 `db:"sp_id"` - SectorID int64 `db:"sector_num"` - RawSize int64 `db:"raw_size"` + ID string `db:"id"` + PieceCid string `db:"piece_cid"` + PieceLen int64 `db:"piece_length"` + PieceOff int64 `db:"piece_offset"` + SPID int64 `db:"sp_id"` + SectorID int64 `db:"sector_num"` + RawSize int64 `db:"raw_size"` + PieceRef sql.NullInt64 `db:"piece_ref"` } var toCheckList []checkEntry err := c.db.Select(ctx, &toCheckList, ` - SELECT mm.piece_cid, mpd.piece_length, mpd.piece_offset, mpd.sp_id, mpd.sector_num, mpd.raw_size + SELECT mm.piece_cid, mpd.piece_length, mpd.piece_offset, mpd.sp_id, mpd.sector_num, mpd.raw_size, mpd.piece_ref, mpd.id FROM market_piece_metadata mm LEFT JOIN market_piece_deal mpd ON mm.piece_cid = mpd.piece_cid AND mm.piece_size = mpd.piece_length WHERE mm.indexed = true @@ -83,18 +97,29 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask return err } - toCheck := make(map[string][]checkEntry) + toCheck := make(map[abi.PieceInfo][]checkEntry) for _, e := range toCheckList { - toCheck[e.PieceCid] = append(toCheck[e.PieceCid], e) + pCid, err := cid.Parse(e.PieceCid) + if err != nil { + return xerrors.Errorf("parsing piece cid: %w", err) + } + pi := abi.PieceInfo{PieceCID: pCid, Size: abi.PaddedPieceSize(e.PieceLen)} + toCheck[pi] = append(toCheck[pi], e) } // Check the number of ongoing indexing tasks var ongoingIndexingTasks int64 - err = c.db.QueryRow(ctx, ` - SELECT COUNT(*) - FROM market_mk12_deal_pipeline - WHERE indexing_created_at IS NOT NULL AND indexed = false - `).Scan(&ongoingIndexingTasks) + err = c.db.QueryRow(ctx, `SELECT + ( + SELECT COUNT(*) + FROM market_mk12_deal_pipeline + WHERE indexing_created_at IS NOT NULL AND indexed = false + ) + + ( + SELECT COUNT(*) + FROM market_mk20_pipeline + WHERE indexing_created_at IS NOT NULL AND indexed = false + ) AS total_pending_indexing;`).Scan(&ongoingIndexingTasks) if err != nil { return xerrors.Errorf("counting ongoing indexing tasks: %w", err) } @@ -105,25 +130,15 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask var have, missing int64 - for p, cent := range toCheck { - pCid, err := cid.Parse(p) - if err != nil { - return xerrors.Errorf("parsing piece cid: %w", err) - } - - pi := abi.PieceInfo{ - PieceCID: pCid, - Size: abi.PaddedPieceSize(cent[0].PieceLen), - } - - commp, err := commcidv2.CommPFromPieceInfo(pi) + for p, cents := range toCheck { + commp, err := commcidv2.CommPFromPieceInfo(p) if err != nil { return xerrors.Errorf("getting piece commP: %w", err) } pieceCid := commp.PCidV2() - // Check if the piece is present in the index store + // Check if the pieceV2 is present in the index store hasEnt, err := c.indexStore.CheckHasPiece(ctx, pieceCid) if err != nil { return xerrors.Errorf("getting piece hash range: %w", err) @@ -134,130 +149,204 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask continue } - // Index not present, flag for repair - missing++ - log.Warnw("piece missing in indexstore", "piece", pieceCid, "task", taskID) - - var uuids []struct { - DealUUID string `db:"uuid"` - } - err = c.db.Select(ctx, &uuids, ` - SELECT uuid - FROM market_mk12_deals - WHERE piece_cid = $1 AND piece_size = $2 - `, pCid.String(), pi.Size) + // Check if the pieceV1 is present in the index store + hasEnt, err = c.indexStore.CheckHasPiece(ctx, p.PieceCID) if err != nil { - return xerrors.Errorf("getting deal uuids: %w", err) - } - if len(uuids) == 0 { - log.Warnw("no deals for unindexed piece", "piece", pieceCid, "task", taskID) - continue - } - - // Check the number of ongoing indexing tasks again - err = c.db.QueryRow(ctx, ` - SELECT COUNT(*) - FROM market_mk12_deal_pipeline - WHERE indexing_created_at IS NOT NULL AND indexed = false - `).Scan(&ongoingIndexingTasks) - if err != nil { - return xerrors.Errorf("counting ongoing indexing tasks: %w", err) - } - if ongoingIndexingTasks >= int64(MaxOngoingIndexingTasks) { - log.Warnw("too many ongoing indexing tasks, stopping processing missing pieces", "task", taskID, "ongoing", ongoingIndexingTasks) - break - } - - // Collect deal UUIDs - dealUUIDs := make([]string, 0, len(uuids)) - for _, u := range uuids { - dealUUIDs = append(dealUUIDs, u.DealUUID) - } - - // Get deal details from market_mk12_deals - var deals []struct { - UUID string `db:"uuid"` - SPID int64 `db:"sp_id"` - PieceCID string `db:"piece_cid"` - PieceSize int64 `db:"piece_size"` - Offline bool `db:"offline"` - URL *string `db:"url"` - Headers []byte `db:"url_headers"` - CreatedAt time.Time `db:"created_at"` - } - err = c.db.Select(ctx, &deals, ` - SELECT uuid, sp_id, piece_cid, piece_size, offline, url, url_headers, created_at - FROM market_mk12_deals - WHERE uuid = ANY($1) - `, dealUUIDs) - if err != nil { - return xerrors.Errorf("getting deal details: %w", err) + return xerrors.Errorf("getting piece hash range: %w", err) } - // Use the first deal for processing - deal := deals[0] - - var sourceSector *storiface.SectorRef - var sourceOff, rawSize int64 - for _, entry := range cent { - if entry.SPID != deal.SPID { - continue - } - if sourceSector = c.findSourceSector(ctx, entry.SPID, entry.SectorID); sourceSector == nil { - // No unsealed copy - continue + if hasEnt { + err = c.indexStore.UpdatePieceCidV1ToV2(ctx, p.PieceCID, pieceCid) + if err != nil { + return xerrors.Errorf("updating piece cid v1 to v2: %w", err) } - sourceOff = entry.PieceOff - rawSize = entry.RawSize - break - } - - if sourceSector == nil { - log.Infow("no unsealed copy of sector found for reindexing", "piece", pieceCid, "task", taskID, "deals", len(deals), "have", have, "missing", missing, "ongoing", ongoingIndexingTasks) + log.Infow("piece cid v1 to v2 updated", "piece", p.PieceCID, "task", taskID) + have++ continue } - var added bool - - _, err = c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - added = false - - // Insert into market_mk12_deal_pipeline - n, err := tx.Exec(` - INSERT INTO market_mk12_deal_pipeline ( - uuid, sp_id, piece_cid, piece_size, raw_size, offline, url, headers, created_at, - sector, sector_offset, reg_seal_proof, - started, after_psd, after_commp, after_find_deal, sealed, complete, - indexed, indexing_created_at, indexing_task_id, should_index - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, - true, true, true, true, true, true, - false, NOW(), NULL, true) - ON CONFLICT (uuid) DO NOTHING - `, deal.UUID, deal.SPID, deal.PieceCID, deal.PieceSize, rawSize, deal.Offline, deal.URL, deal.Headers, deal.CreatedAt, - sourceSector.ID.Number, sourceOff, int64(sourceSector.ProofType)) + // Index not present, flag for repair + missing++ + log.Warnw("piece missing in indexstore", "piece", pieceCid, "task", taskID) + + for _, cent := range cents { + var isMK12 bool + var id ulid.ULID + + id, err := ulid.Parse(cent.ID) if err != nil { - return false, xerrors.Errorf("upserting into deal pipeline for uuid %s: %w", deal.UUID, err) + serr := err + _, err = uuid.Parse(cent.ID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w, %w", serr, err) + } + isMK12 = true } - if n == 0 { - return false, nil - } - added = true - _, err = tx.Exec(`UPDATE market_piece_metadata SET indexed = FALSE WHERE piece_cid = $1`, pieceCid.String()) - if err != nil { - return false, xerrors.Errorf("updating market_piece_metadata.indexed column: %w", err) + var scheduled bool + + if isMK12 { + // Get deal details from market_mk12_deals + var mk12deals []struct { + UUID string `db:"uuid"` + SPID int64 `db:"sp_id"` + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + Offline bool `db:"offline"` + CreatedAt time.Time `db:"created_at"` + } + err = c.db.Select(ctx, &mk12deals, `SELECT + uuid, + sp_id, + piece_cid, + piece_size, + offline, + created_at, + FALSE AS ddo + FROM market_mk12_deals + WHERE uuid = $1 + + UNION ALL + + SELECT + uuid, + sp_id, + piece_cid, + piece_size, + TRUE AS offline, + created_at, + TRUE AS ddo + FROM market_direct_deals + WHERE uuid = $1; + `, cent.ID) + if err != nil { + return xerrors.Errorf("getting deal details: %w", err) + } + + if len(mk12deals) == 0 { + log.Warnw("no mk12 deals for unindexed piece", "piece", pieceCid, "task", taskID) + continue + } + + mk12deal := mk12deals[0] + + if cent.PieceRef.Valid { + continue // This is mk20 deal + } + if cent.SPID != mk12deal.SPID { + continue + } + sourceSector := c.findSourceSector(ctx, cent.SPID, cent.SectorID) + if sourceSector == nil { + continue + } + + var added bool + + _, err = c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + added = false + + // Insert into market_mk12_deal_pipeline + n, err := tx.Exec(` + INSERT INTO market_mk12_deal_pipeline ( + uuid, sp_id, piece_cid, piece_size, raw_size, offline, created_at, + sector, sector_offset, reg_seal_proof, + started, after_psd, after_commp, after_find_deal, sealed, complete, + indexed, indexing_created_at, indexing_task_id, should_index + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + true, true, true, true, true, true, + false, NOW(), NULL, true) + ON CONFLICT (uuid) DO NOTHING + `, mk12deal.UUID, mk12deal.SPID, mk12deal.PieceCID, mk12deal.PieceSize, cent.RawSize, mk12deal.Offline, mk12deal.CreatedAt, + sourceSector.ID.Number, cent.PieceOff, int64(sourceSector.ProofType)) + if err != nil { + return false, xerrors.Errorf("upserting into deal pipeline for uuid %s: %w", mk12deal.UUID, err) + } + if n == 0 { + return false, nil + } + added = true + + _, err = tx.Exec(`UPDATE market_piece_metadata SET indexed = FALSE WHERE piece_cid = $1 AND piece_size = $2`, p.PieceCID.String(), p.Size) + if err != nil { + return false, xerrors.Errorf("updating market_piece_metadata.indexed column: %w", err) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("inserting into market_mk12_deal_pipeline: %w", err) + } + + if added { + log.Infow("added reindexing pipeline entry", "uuid", mk12deal.UUID, "task", taskID, "piece", pieceCid) + ongoingIndexingTasks++ + scheduled = true + } + } else { + if !cent.PieceRef.Valid { + continue + } + + deal, err := mk20.DealFromDB(ctx, c.db, id) + if err != nil { + log.Warnw("failed to get deal from db", "id", id.String(), "task", taskID) + continue + } + + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return xerrors.Errorf("parsing provider address: %w", err) + } + + rawSize, err := deal.Data.RawSize() + if err != nil { + return xerrors.Errorf("getting raw size: %w", err) + } + + if uint64(cent.SPID) != spid { + continue + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", cent.PieceRef.Int64), + } + + data := deal.Data + ddo := deal.Products.DDOV1 + + aggregation := 0 + if data.Format.Aggregate != nil { + aggregation = int(data.Format.Aggregate.Type) + } + + n, err := c.db.Exec(ctx, `INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, piece_size, raw_size, + offline, url, indexing, announce, duration, piece_aggregation, + started, downloaded, after_commp, aggregated, sector, reg_seal_proof, sector_offset, sealed, + indexing_created_at, complete) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, + TRUE, TRUE, True, True, $14, 0, $15 TRUE, NOW(), TRUE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created + deal.Identifier.String(), spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(rawSize), + false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, + cent.SectorID, cent.PieceOff) + if err != nil { + return xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + log.Infow("added reindexing pipeline entry", "id", id, "task", taskID, "piece", pieceCid) + ongoingIndexingTasks++ + scheduled = true } - return true, nil - }, harmonydb.OptionRetry()) - if err != nil { - return xerrors.Errorf("inserting into market_mk12_deal_pipeline: %w", err) - } + if scheduled { + break // Break out of PieceDeal loop + } - if added { - log.Infow("added reindexing pipeline entry", "uuid", deal.UUID, "task", taskID, "piece", deal.PieceCID) - ongoingIndexingTasks++ } if ongoingIndexingTasks >= int64(MaxOngoingIndexingTasks) { @@ -336,9 +425,9 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas return nil } - var have, missisg, issues int64 + var have, missing, issues int64 defer func() { - log.Infow("IPNI Ad check", "have", have, "missisg", missisg, "issues", issues, "err", err) + log.Infow("IPNI Ad check", "have", have, "missing", missing, "issues", issues, "err", err) }() for _, deal := range toCheck { @@ -429,7 +518,7 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas continue } - missisg++ + missing++ n, err := c.db.Exec(ctx, ` INSERT INTO market_mk12_deal_pipeline ( @@ -463,6 +552,189 @@ func (c *CheckIndexesTask) checkIPNI(ctx context.Context, taskID harmonytask.Tas return nil } +func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask.TaskID) (err error) { + var ids []struct { + ID string `db:"id"` + } + + err = c.db.Select(ctx, &ids, `SELECT m.id + FROM market_mk20_deal AS m + LEFT JOIN ipni AS i + ON m.piece_cid = i.piece_cid + AND m.piece_size = i.piece_size + LEFT JOIN market_mk20_pipeline AS p + ON m.id = p.id + LEFT JOIN market_mk20_pipeline_waiting AS w + ON m.id = w.id + WHERE m.ddo_v1->>'announce_to_ipni' = 'true' + AND i.piece_cid IS NULL + AND p.id IS NULL + AND w.id IS NULL;`) + if err != nil { + return xerrors.Errorf("getting mk20 deals which are not announced: %w", err) + } + + if len(ids) == 0 { + return nil + } + + var ipniPeerIDs []struct { + SpID int64 `db:"sp_id"` + PeerID string `db:"peer_id"` + } + err = c.db.Select(ctx, &ipniPeerIDs, `SELECT sp_id, peer_id FROM ipni_peerid`) + if err != nil { + return xerrors.Errorf("getting ipni tasks: %w", err) + } + + spToPeer := map[int64]string{} + for _, d := range ipniPeerIDs { + spToPeer[d.SpID] = d.PeerID + } + + var ongoingIpniTasks int64 + err = c.db.QueryRow(ctx, `SELECT COUNT(1) FROM ipni_task`).Scan(&ongoingIpniTasks) + if err != nil { + return xerrors.Errorf("getting ipni tasks: %w", err) + } + if ongoingIpniTasks >= int64(MaxOngoingIndexingTasks) { + log.Debugw("too many ongoing ipni tasks, skipping ipni index checks", "task", taskID, "ongoing", ongoingIpniTasks) + return nil + } + + var have, missing, issues int64 + for _, i := range ids { + id, err := ulid.Parse(i.ID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + + deal, err := mk20.DealFromDB(ctx, c.db, id) + if err != nil { + return xerrors.Errorf("getting deal from db: %w", err) + } + + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return xerrors.Errorf("parsing provider address: %w", err) + } + + pi := abi.PieceInfo{ + PieceCID: deal.Data.PieceCID, + Size: deal.Data.Size, + } + + commp, err := commcidv2.CommPFromPieceInfo(pi) + if err != nil { + return xerrors.Errorf("getting commp from PieceInfo: %w", err) + } + + pcid := commp.PCidV2() + + var ctxIdBuf bytes.Buffer + err = pi.MarshalCBOR(&ctxIdBuf) + if err != nil { + return xerrors.Errorf("marshaling piece info: %w", err) + } + + ctxId := ctxIdBuf.Bytes() + + provider, ok := spToPeer[int64(spid)] + if !ok { + issues++ + log.Warnw("no peer id for spid", "spid", spid, "checkPiece", pcid) + continue + } + + var hasEnt int64 + err = c.db.QueryRow(ctx, `SELECT count(1) FROM ipni WHERE context_id=$1 AND provider=$2`, ctxId, provider).Scan(&hasEnt) + if err != nil { + return xerrors.Errorf("getting piece hash range: %w", err) + } + if hasEnt > 0 { + // has the entry + have++ + continue + } + + hasIndex, err := c.indexStore.CheckHasPiece(ctx, pcid) + if err != nil { + return xerrors.Errorf("getting piece hash range: %w", err) + } + if !hasIndex { + log.Warnw("no index for piece with missing IPNI Ad", "piece", pcid, "checkPiece", pi.PieceCID) + issues++ + continue + } + + var sourceSector []struct { + SectorNum int64 `db:"sector_num"` + PieceOffset int64 `db:"piece_offset"` + RawSize int64 `db:"raw_size"` + PieceRef sql.NullInt64 `db:"piece_ref"` + } + err = c.db.Select(ctx, &sourceSector, `SELECT sector_num, piece_offset, raw_size, piece_ref FROM market_piece_deal WHERE id = $1`, id.String()) + if err != nil { + return xerrors.Errorf("getting source sector: %w", err) + } + if len(sourceSector) == 0 { + log.Warnw("no source sector for piece", "piece", pcid, "checkPiece", pi.PieceCID) + issues++ + continue + } + + src := sourceSector[0] + + if !src.PieceRef.Valid { + log.Warnw("no piece ref for ipni reindexing", "piece", pi.PieceCID, "checkPiece", pcid) + missing++ + continue + } + + rawSize, err := deal.Data.RawSize() + if err != nil { + return xerrors.Errorf("getting raw size: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", src.PieceRef.Int64), + } + + data := deal.Data + ddo := deal.Products.DDOV1 + + aggregation := 0 + if data.Format.Aggregate != nil { + aggregation = int(data.Format.Aggregate.Type) + } + + n, err := c.db.Exec(ctx, `INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, piece_size, raw_size, + offline, url, indexing, announce, duration, piece_aggregation, + started, downloaded, after_commp, aggregated, sector, reg_seal_proof, sector_offset, sealed, + indexing_created_at, indexed, complete) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, + TRUE, TRUE, True, True, $14, 0, $15, TRUE, NOW(), TRUE, FALSE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created + deal.Identifier.String(), spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(rawSize), + false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, + src.SectorNum, src.PieceOffset) + if err != nil { + return xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + log.Infow("created IPNI reindexing pipeline", "piece", pi.PieceCID, "spid", spid) + ongoingIpniTasks++ + if ongoingIpniTasks >= int64(MaxOngoingIndexingTasks) { + return nil + } + } + + return nil +} + func (c *CheckIndexesTask) findSourceSector(ctx context.Context, spid, sectorNum int64) *storiface.SectorRef { var sourceSector *storiface.SectorRef var qres []struct { diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index 7391dd553..b4491c5c5 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -3,6 +3,7 @@ package indexing import ( "bufio" "context" + "database/sql" "errors" "fmt" "io" @@ -35,6 +36,7 @@ import ( "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/pieceprovider" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/mk20" @@ -45,6 +47,7 @@ var log = logging.Logger("indexing") type IndexingTask struct { db *harmonydb.DB indexStore *indexstore.IndexStore + pieceProvider *pieceprovider.SectorReader cpr *cachedreader.CachedPieceReader sc *ffi.SealCalls cfg *config.CurioConfig @@ -53,11 +56,12 @@ type IndexingTask struct { max taskhelp.Limiter } -func NewIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IndexingTask { +func NewIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.SectorReader, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IndexingTask { return &IndexingTask{ db: db, indexStore: indexStore, + pieceProvider: pieceProvider, cpr: cpr, sc: sc, cfg: cfg, @@ -76,12 +80,14 @@ type itask struct { Size abi.PaddedPieceSize `db:"piece_size"` Offset int64 `db:"sector_offset"` RawSize int64 `db:"raw_size"` + Url sql.NullString `db:"url"` ShouldIndex bool `db:"should_index"` IndexingCreatedAt time.Time `db:"indexing_created_at"` Announce bool `db:"announce"` ChainDealId abi.DealID `db:"chain_deal_id"` IsDDO bool `db:"is_ddo"` Mk20 bool `db:"mk20"` + PieceRef int64 } func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { @@ -99,6 +105,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do p.sector_offset, p.reg_seal_proof, p.raw_size, + p.url, p.should_index, p.announce, p.is_ddo, @@ -126,6 +133,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do sector_offset, reg_seal_proof, raw_size, + url, indexing as should_index, announce, TRUE AS is_ddo, @@ -174,6 +182,26 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do if deal.Data.Format.Raw != nil { byteData = true } + + if !task.Url.Valid { + return false, xerrors.Errorf("no url for mk20 deal") + } + + url, err := url.Parse(task.Url.String) + if err != nil { + return false, xerrors.Errorf("parsing url: %w", err) + } + + if url.Scheme != "pieceref" { + return false, xerrors.Errorf("invalid url scheme: %s", url.Scheme) + } + + refNum, err := strconv.ParseInt(url.Opaque, 10, 64) + if err != nil { + return false, xerrors.Errorf("parsing piece reference number: %w", err) + } + + task.PieceRef = refNum } // Return early if already indexed or should not be indexed @@ -198,10 +226,26 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do } pc2 := commp.PCidV2() - reader, _, err := i.cpr.GetSharedPieceReader(ctx, pc2) + var reader storiface.Reader - if err != nil { - return false, xerrors.Errorf("getting piece reader: %w", err) + if task.Mk20 { + reader, _, err = i.cpr.GetSharedPieceReader(ctx, pc2) + + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + } else { + reader, err = i.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(task.SpID), + Number: task.Sector, + }, + ProofType: task.Proof, + }, storiface.PaddedByteIndex(task.Offset).Unpadded(), task.Size.Unpadded(), pieceCid) + + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } } defer reader.Close() @@ -462,6 +506,7 @@ func IndexAggregate(pieceCid cid.Cid, } aggidx := make(map[cid.Cid][]datasegment.SegmentDesc) + aggidx[pieceCid] = valid log.Infow("Indexing aggregate", "piece_size", size, "num_chunks", len(valid), "num_sub_pieces", len(subPieces)) @@ -531,10 +576,18 @@ func IndexAggregate(pieceCid cid.Cid, // recordCompletion add the piece metadata and piece deal to the DB and // records the completion of an indexing task in the database func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID harmonytask.TaskID, indexed bool) error { - _, err := i.db.Exec(ctx, `SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - task.UUID, task.PieceCid, !task.IsDDO, task.SpID, task.Sector, task.Offset, task.Size, task.RawSize, indexed, false, task.ChainDealId) - if err != nil { - return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) + if task.Mk20 { + _, err := i.db.Exec(ctx, `SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + task.UUID, task.PieceCid, !task.IsDDO, task.SpID, task.Sector, task.Offset, task.Size, task.RawSize, indexed, task.PieceRef, false, task.ChainDealId) + if err != nil { + return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) + } + } else { + _, err := i.db.Exec(ctx, `SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + task.UUID, task.PieceCid, !task.IsDDO, task.SpID, task.Sector, task.Offset, task.Size, task.RawSize, indexed, false, task.ChainDealId) + if err != nil { + return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) + } } // If IPNI is disabled then mark deal as complete otherwise just mark as indexed diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index bd3ae2349..1801882a5 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -37,6 +37,8 @@ import ( "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/pieceprovider" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/ipni/chunker" "github.com/filecoin-project/curio/market/ipni/ipniculib" @@ -46,22 +48,24 @@ import ( var ilog = logging.Logger("ipni") type IPNITask struct { - db *harmonydb.DB - indexStore *indexstore.IndexStore - cpr *cachedreader.CachedPieceReader - sc *ffi.SealCalls - cfg *config.CurioConfig - max taskhelp.Limiter + db *harmonydb.DB + indexStore *indexstore.IndexStore + pieceProvider *pieceprovider.SectorReader + cpr *cachedreader.CachedPieceReader + sc *ffi.SealCalls + cfg *config.CurioConfig + max taskhelp.Limiter } -func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { +func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.SectorReader, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { return &IPNITask{ - db: db, - indexStore: indexStore, - cpr: cpr, - sc: sc, - cfg: cfg, - max: max, + db: db, + indexStore: indexStore, + pieceProvider: pieceProvider, + cpr: cpr, + sc: sc, + cfg: cfg, + max: max, } } @@ -120,11 +124,23 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, xerrors.Errorf("getting piece commP: %w", err) } - reader, _, err := I.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) - + // Try to read unsealed sector first (mk12 deal) + reader, err := I.pieceProvider.ReadPiece(ctx, storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(task.SPID), + Number: task.Sector, + }, + ProofType: task.Proof, + }, storiface.PaddedByteIndex(task.Offset).Unpadded(), pi.Size.Unpadded(), pi.PieceCID) if err != nil { - return false, xerrors.Errorf("getting piece reader: %w", err) + serr := err + // Try to read piece (mk20 deal) + reader, _, err = I.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) + if err != nil { + return false, xerrors.Errorf("getting piece reader from sector and piece park: %w, %w", serr, err) + } } + defer reader.Close() var isMK20 bool diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index b76b4d2b7..78cfbc0b6 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -623,6 +623,7 @@ type PieceDeal struct { Length int64 `db:"piece_length" json:"length"` RawSize int64 `db:"raw_size" json:"raw_size"` Miner string `json:"miner"` + MK20 bool `db:"-" json:"mk20"` } type PieceInfo struct { @@ -681,6 +682,10 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er if err != nil { return nil, err } + _, err = uuid.Parse(pieceDeals[i].ID) + if err != nil { + pieceDeals[i].MK20 = true + } pieceDeals[i].Miner = addr.String() ret.Size = pieceDeals[i].Length } diff --git a/web/static/pages/actor/actor-detail.mjs b/web/static/pages/actor/actor-detail.mjs index 63b57692f..15f1e2f69 100644 --- a/web/static/pages/actor/actor-detail.mjs +++ b/web/static/pages/actor/actor-detail.mjs @@ -99,7 +99,9 @@ customElements.define('actor-detail', class Actor extends LitElement { Source Config Layers: - ${actorInfo.Summary.CLayers} + + ${entry.CLayers.map(layer => html`${layer} `)} + Sector Size: diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 24f14045b..137baf2bb 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -120,25 +120,6 @@ class DealDetails extends LitElement { `; } - renderAggregateSubs(subs) { - if (!subs?.length) return ''; - return html` - - - - ${subs.map((s, i) => html` - - - - - - - `)} - -
    #CarRawAggregate
    ${i + 1}${s.car ? 'Yes' : ''}${s.raw ? 'Yes' : ''}${s.aggregate ? 'Yes' : ''}
    - `; - } - renderSourceHTTP(src) { return html` @@ -179,22 +160,26 @@ class DealDetails extends LitElement { renderSourceAggregate(src) { return html` - ${src.pieces.map((piece, i) => html` -
    - Piece ${i + 1}: ${piece.piece_cid['/']}
    - [DETAILS] - 
    -
    - - - - -
    PieceCID${piece.piece_cid['/']}
    Size${piece.piece_size}
    Format${this.renderPieceFormat(piece.format)}
    Source${this.renderDataSource(piece)}
    - + [Aggregate Details] +
    + ${src.pieces.map((piece, i) => html` +
    + + Piece ${i + 1} + + + + + + +
    PieceCID${piece.piece_cid['/']}
    Size${piece.piece_size}
    ${this.renderPieceFormat(piece.format)}
    ${this.renderDataSource(piece)}
    + + +
    + `)} +
    -
    • - `)} `; } diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index c0e6a6605..b522f0e93 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -126,7 +126,11 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { ${this.data.deals.map((item) => html` - ${item.id} + + ${item.mk20 + ? html`${item.id}` + : html`${item.id}`} + ${item.boost_deal ? 'Boost' : (item.legacy_deal ? 'Legacy' : 'DDO')} ${item.miner} ${item.chain_deal_id} @@ -391,32 +395,32 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { Offline URL${entry.mk20_pipeline.url.Valid ? entry.mk20_pipeline.url.String : 'N/A'} Headers
    ${JSON.stringify(entry.mk20_pipeline.headers, null, 2)}
    - Should Index${this.renderNullableYesNo(entry.mk20_pipeline.indexing.Bool)} + Should Index${this.renderNullableYesNo(entry.mk20_pipeline.indexing)} Announce - ${this.renderNullableYesNo(entry.mk20_pipeline.announce.Bool)} + ${this.renderNullableYesNo(entry.mk20_pipeline.announce)}
    Progress šŸ› ļø
    Data Fetched - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.started.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.downloaded)} After Commp - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.after_commp.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.after_commp)} Aggregated - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.aggregated.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.aggregated)} Sealed - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.sealed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.sealed)} Indexed - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.indexed.Bool)} + ${this.renderNullableDoneNotDone(entry.mk20_pipeline.indexed)} Announced From 34b12f80ddee8933840c79f9570058ef7bd6f26b Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 28 May 2025 20:25:00 +0400 Subject: [PATCH 11/55] update pdp client service url --- cmd/pdptool/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/pdptool/main.go b/cmd/pdptool/main.go index 6fa383f68..1fb266721 100644 --- a/cmd/pdptool/main.go +++ b/cmd/pdptool/main.go @@ -195,6 +195,8 @@ var pingCmd = &cli.Command{ return errCreateToken } + serviceURL = serviceURL + "/market" + // Append /pdp/ping to the service URL pingURL := serviceURL + "/pdp/ping" @@ -512,6 +514,7 @@ var pieceUploadCmd = &cli.Command{ } serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" jwtToken := cctx.String("jwt-token") notifyURL := cctx.String("notify-url") serviceName := cctx.String("service-name") @@ -669,6 +672,7 @@ var uploadFileCmd = &cli.Command{ } serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" jwtToken := cctx.String("jwt-token") serviceName := cctx.String("service-name") hashType := cctx.String("hash-type") @@ -866,6 +870,7 @@ var createProofSetCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" serviceName := cctx.String("service-name") recordKeeper := cctx.String("recordkeeper") extraDataHexStr := cctx.String("extra-data") @@ -961,6 +966,7 @@ var getProofSetStatusCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" serviceName := cctx.String("service-name") txHash := cctx.String("tx-hash") @@ -1072,6 +1078,7 @@ var getProofSetCmd = &cli.Command{ } serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" serviceName := cctx.String("service-name") // Load the private key @@ -1178,6 +1185,7 @@ var addRootsCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" serviceName := cctx.String("service-name") proofSetID := cctx.Uint64("proof-set-id") rootInputs := cctx.StringSlice("root") @@ -1422,6 +1430,7 @@ var removeRootsCmd = &cli.Command{ }, Action: func(cctx *cli.Context) error { serviceURL := cctx.String("service-url") + serviceURL = serviceURL + "/market" serviceName := cctx.String("service-name") proofSetID := cctx.Uint64("proof-set-id") rootID := cctx.Uint64("root-id") From f10b1c0743d62185aa5be3a1bd152d187d01286f Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 29 May 2025 20:55:45 +0400 Subject: [PATCH 12/55] basic docs, new api, UI changes --- deps/config/doc_gen.go | 6 + deps/config/types.go | 4 + documentation/en/SUMMARY.md | 4 + .../default-curio-configuration.md | 5 + documentation/en/market-2.0/README.md | 192 +++++ documentation/en/market-2.0/api.md | 209 ++++++ documentation/en/market-2.0/contracts.md | 119 +++ documentation/en/market-2.0/products.md | 246 +++++++ .../harmonydb/sql/20250505-market_mk20.sql | 1 - market/mk20/http/http.go | 158 +++- market/mk20/http/info.md | 43 ++ market/mk20/http/test.html | 692 ------------------ market/mk20/mk20.go | 39 +- market/mk20/mk20_utils.go | 134 +++- market/mk20/mk20gen/gen.go | 18 + market/mk20/utils.go | 12 + web/api/webrpc/ipni.go | 3 +- web/api/webrpc/market_20.go | 245 +++++++ web/static/pages/mk20-deal/deal.mjs | 62 +- web/static/pages/mk20-deal/index.html | 1 + web/static/pages/mk20/deal-search.mjs | 68 ++ web/static/pages/mk20/index.html | 20 + web/static/pages/mk20/settings.mjs | 297 ++++++++ 23 files changed, 1785 insertions(+), 793 deletions(-) create mode 100644 documentation/en/market-2.0/README.md create mode 100644 documentation/en/market-2.0/api.md create mode 100644 documentation/en/market-2.0/contracts.md create mode 100644 documentation/en/market-2.0/products.md delete mode 100644 market/mk20/http/test.html create mode 100644 web/static/pages/mk20/deal-search.mjs create mode 100644 web/static/pages/mk20/settings.mjs diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 6e0fa47e8..3addd52c1 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -1091,6 +1091,12 @@ When the cumulative size of all deals in process reaches this number, new deals Comment: `DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list If True then all deals coming from unknown clients will be rejected. (Default: false)`, }, + { + Name: "MaxParallelUploads", + Type: "int", + + Comment: `MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16)`, + }, }, "MarketConfig": { { diff --git a/deps/config/types.go b/deps/config/types.go index 35eb68f86..a4eb97894 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -113,6 +113,7 @@ func DefaultCurioConfig() *CurioConfig { MK20: MK20Config{ ExpectedPoRepSealDuration: 8 * time.Hour, ExpectedSnapSealDuration: 2 * time.Hour, + MaxParallelUploads: 16, }, IPNI: IPNIConfig{ ServiceURL: []string{"https://cid.contact"}, @@ -908,4 +909,7 @@ type MK20Config struct { // DenyUnknownClients determines the default behaviour for the deal of clients which are not in allow/deny list // If True then all deals coming from unknown clients will be rejected. (Default: false) DenyUnknownClients bool + + // MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16) + MaxParallelUploads int } diff --git a/documentation/en/SUMMARY.md b/documentation/en/SUMMARY.md index 5306f59da..ce13b3f8d 100644 --- a/documentation/en/SUMMARY.md +++ b/documentation/en/SUMMARY.md @@ -27,6 +27,10 @@ * [Market UI](curio-market/market-ui.md) * [Retrievals](curio-market/retrievals.md) * [Migrating From Boost](curio-market/migrating-from-boost.md) +* [Market 2.0](market-2.0/README.md) + * [Products](market-2.0/products.md) + * [Market Contracts](market-2.0/contracts.md) + * [API](market-2.0/api.md) * [Snap Deals](snap-deals.md) * [Batch Sealing with SupraSeal](supraseal.md) * [Scaling Curio cluster](scaling-curio-cluster.md) diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 932a488fa..28597493b 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -699,6 +699,11 @@ description: The default curio configuration # type: bool #DenyUnknownClients = false + # MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16) + # + # type: int + #MaxParallelUploads = 16 + # IPNI configuration for ipni-provider # # type: IPNIConfig diff --git a/documentation/en/market-2.0/README.md b/documentation/en/market-2.0/README.md new file mode 100644 index 000000000..6da4cd306 --- /dev/null +++ b/documentation/en/market-2.0/README.md @@ -0,0 +1,192 @@ +# Market 2.0 + +This guide introduces the new Filecoin Market 2.0 architecture for clients, developers, and aggregators. It explains how to use the new modular, contract-governed storage market and how to interact with Curio-based storage providers under this new system. + +--- + +## šŸ§­ Overview + +Filecoin's Market 2.0 removes legacy assumptions of the built-in storage market actor. Instead, deals are processed through **user-defined smart contracts**, allowing: + +* Flexible pricing and service terms +* Support for custom retrieval logic +* Contract-governed deal lifecycle +* Composability via extensible "products" + +Curio's role is purely to onboard data and respect contract termsā€”it does not mediate pricing, payments, or retrieval policy. + +--- + +## šŸ“” Supported Endpoints + +### šŸ”„ POST `/market/mk20/store` + +Accept a new deal (JSON body). + +* Auto-validates structure, products, sources, contract +* If valid, returns `200 OK` +* Otherwise returns appropriate error code (e.g. `400`, `422`, etc) + +### šŸ§¾ GET `/market/mk20/status?id=` + +Check the status of a deal. + +* Returns one of: `accepted`, `processing`, `sealing`, `indexing`, `complete`, or `failed` + +### šŸ—‚ PUT `/market/mk20/data?id=` + +Used only when `source_httpput` is selected. + +* Clients upload raw bytes +* `Content-Length` must match raw size + +### šŸ“œ GET `/market/mk20/contracts` + +Return list of supported contract addresses + +### šŸ§  GET `/market/mk20/info` + +Markdown documentation of deal format and validation rules + +### šŸ§  GET `/market/mk20/producs` + +Json list of products supported by the provider + +### šŸ§  GET `/market/mk20/sources` + +Json list of data sources supported by the provider + +--- + +## šŸ§‘ā€šŸ’» Clients + +### šŸ“ Submitting a Deal + +Clients submit a deal to a Curio node using the `/market/mk20/store` endpoint. A deal includes: + +* A unique ULID identifier +* A `DataSource` (e.g. HTTP, offline, PUT) +* One or more `Products` (like `ddov1`) that define how the deal should behave + +#### Example Products: + +* `ddov1`: governs how the data should be stored and verified +* (future) `aclv1`: may define retrieval access controls +* (future) `retrievalv1`: may define retrieval SLA or payment terms + +### šŸ›  Smart Contract Control + +Clients must select a contract that: + +* Is supported by the SP +* Implements a deal validation method that returns a valid DealID + +Clients pass the contract address, method name, and encoded params. + +### šŸ” Deal Lifecycle + +The contract governs whether the deal is valid. If valid: + +* The SP accepts and starts onboarding the data +* The deal may be indexed and/or announced to IPNI, based on deal config +* Data may be retrieved later via PieceCIDv2, PayloadCID, or subpiece CID + +--- + +## šŸ§± Developers + +### šŸ§© Building New Products + +Each product is a self-contained struct in the deal payload. Developers can: + +* Define new product types (e.g., `aclv1`, `retrievalv1`, `auditv1`) +* Implement validation logic on the SP side +* Optionally affect indexing, retrievals, ACLs, or other lifecycle aspects + +This makes the deal structure extensible **without requiring protocol or DB changes.** + +### šŸ§  Writing Market Contracts + +A contract must: + +* Be added to the SP's whitelist +* Implement a method (e.g. `verifyDeal`) that takes a single `bytes` parameter +* Return a valid deal ID if the deal is accepted + +Contracts can implement features like: + +* Off-chain or on-chain ACL logic +* Multi-party deal approval +* FIL+ verification +* SLA enforcement + +--- + +## šŸ” Aggregators + +Data aggregators in Market 2.0 should: + +* No longer implement protocol-level workarounds (like ACLs or approvals) +* Provide value-added services like dashboards, alerts, analytics, SDKs +* Optionally act as data sources for multi-client deal generation + +Market 2.0 aims to reduce dependency on aggregators by letting providers and contracts do the heavy lifting. + +--- + +## šŸ“¦ Retrievals + +Curio supports the following retrieval inputs: + +* **PieceCIDv2**: required for all piece-level retrievals +* **PayloadCID**: if indexing is enabled +* **Subpiece CID**: if the deal was aggregated and subpieces were indexed + +ACL-based gating is not yet implemented, but future products can enable it. + +--- + +## ā™»ļø Deal Lifecycle in Curio + +1. **Client submits** deal with products, data, and contract call info +2. **Curio validates** all inputs and uses the contract to get a DealID +3. **Data is onboarded** via HTTP, offline import, PUT, or aggregation +4. **Products control** indexing, IPNI, and future extensibility +5. **Data is removed** from disk and DB when the sector expires + +--- + +## šŸ§Ŗ Current Product: `ddov1` + +This product represents the first non-native Filecoin market product. It includes: + +* Provider, client, and piece manager addresses +* Optional AllocationID (or minimum duration) +* Contract + verification method for DealID +* Indexing and IPNI flags +* Notification hooks for deal lifecycle events + +More details can be found in the product schema or SP guide. + +--- + +## šŸ”® Future Directions + +* ACL enforcement via `aclv1` +* Retrieval policy enforcement via `retrievalv1` +* Sealed sector access / download pricing +* Smart contract SLAs and renewals +* Market UIs and dashboards + +--- + +## āœ… Summary + +Market 2.0 enables: + +* Composable, contract-governed storage deals +* Modular product design +* Client-friendly HTTP-first onboarding +* Decoupled market innovation from SP software +* Stronger integration paths for aggregators and external tools + diff --git a/documentation/en/market-2.0/api.md b/documentation/en/market-2.0/api.md new file mode 100644 index 000000000..5e23cbff5 --- /dev/null +++ b/documentation/en/market-2.0/api.md @@ -0,0 +1,209 @@ +# MK20 API Reference + +This document describes the HTTP endpoints supported by the Market 2.0 module in Curio. These endpoints are used by clients and external systems to submit storage deals, upload data, track status, and fetch provider configuration. + +--- + +## šŸŒ Base Path + +All endpoints are exposed under: + +``` +/market/mk20 +``` + +--- + +## šŸ”„ POST `/store` + +Submit a new MK20 deal to the storage provider. + +* **Content-Type**: `application/json` +* **Body**: JSON-encoded Deal +* **Query Parameters**: None + +### āœ… Response + +* `200 OK` ā€“ Deal accepted successfully +* `400 Bad Request` ā€“ Malformed JSON or missing required fields +* `422 Unprocessable Entity` ā€“ Unsupported product or data source +* `426` ā€“ Deal rejected by contract +* `430` ā€“ Malformed data source +* `441` ā€“ Deal duration too short + +### šŸ§Ŗ Example + +```http +POST /market/mk20/store +Content-Type: application/json + +{ + "identifier": "01H9Y...", + "data": { ... }, + "products": { ... } +} +``` + +--- + +## šŸ§¾ GET `/status` + +Retrieve the current processing status of a deal. + +* **Query Parameters**: + + * `id`: ULID of the deal + +### āœ… Response + +* `200 OK`: Returns JSON-encoded status +* `400 Bad Request`: Missing or malformed `id` +* `404 Not Found`: No such deal +* `500 Internal Server Error`: Unexpected backend error + +### šŸ“„ Response Schema + +```json +{ + "status": "accepted" | "processing" | "sealing" | "indexing" | "complete" | "failed", + "error_msg": "string (optional)" +} +``` + +--- + +## šŸ—‚ PUT `/data` + +Upload raw deal data for deals that declared a `source_httpput` source. + +* **Headers**: + + * `Content-Type: application/octet-stream` + * `Content-Length`: must match declared raw size +* **Query Parameters**: + + * `id`: ULID of the deal +* **Body**: Raw byte stream + +### āœ… Response + +* `200 OK`: Data accepted +* `400 Bad Request`: Invalid/missing content headers +* `413 Payload Too Large`: Content exceeds allowed size +* `415 Unsupported Media Type`: Incorrect content type + +--- + +## šŸ“œ GET `/contracts` + +Return a list of smart contract addresses currently whitelisted by the provider. + +### āœ… Response + +```json +{ + "contracts": [ + "0x123...", + "0xabc..." + ] +} +``` + +* `200 OK`: List of contracts +* `500 Internal Server Error`: Failure fetching contract list + +--- + +## šŸ§  GET `/info` + +Returns markdown-formatted documentation describing: + +* Supported deal structure +* Data source formats +* Product extensions + +### āœ… Response + +* `200 OK`: Markdown string +* `500 Internal Server Error`: If the info file cannot be generated + +--- + +### šŸ§° GET `/products` + +Fetch a JSON list of supported deal products enabled on this provider. + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A + +#### āœ… Response +- `200 OK`: JSON array of enabled products +- `500 Internal Server Error`: If the list cannot be fetched + +#### šŸ§Ŗ Example Response +```json +{ + "products": [ + "ddo_v1", + "aclv1" + ] +} +``` + +--- + +### šŸŒ GET `/sources` + +Fetch a JSON list of supported data source types enabled on this provider. + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A + +#### āœ… Response +- `200 OK`: JSON array of enabled data sources +- `500 Internal Server Error`: If the list cannot be fetched + +#### šŸ§Ŗ Example Response +```json +{ + "sources": [ + "http", + "offline", + "put", + "aggregate" + ] +} +``` + +--- + +## šŸ“‘ Error Code Summary + +| Code | Meaning | +| ---- | ---------------------------------- | +| 200 | Success | +| 400 | Bad proposal or malformed JSON | +| 422 | Unsupported product or data source | +| 426 | Deal rejected by contract | +| 430 | Malformed data source | +| 441 | Duration too short | +| 500 | Internal server error | + +--- + +## šŸ§© Status Code Values (from `/status`) + +| Value | Meaning | +| ------------ | ----------------------------------------------- | +| `accepted` | Deal was accepted and is waiting for processing | +| `processing` | Deal is being staged or fetched | +| `sealing` | Deal is being sealed into a sector | +| `indexing` | Deal is being indexed for CID-based retrievals | +| `complete` | Deal has been sealed and is finalized | +| `failed` | Deal failed at any point in the pipeline | + +--- + +For full type schemas, see the `/info` endpoint or consult the documentation. diff --git a/documentation/en/market-2.0/contracts.md b/documentation/en/market-2.0/contracts.md new file mode 100644 index 000000000..757ec2364 --- /dev/null +++ b/documentation/en/market-2.0/contracts.md @@ -0,0 +1,119 @@ +# Smart Contract Integration Guide + +This guide explains how to write, deploy, and integrate a smart contract that governs storage deals in the Market 2.0 architecture. Contracts are responsible for determining whether a deal is valid and returning a DealID. + +--- + +## šŸŽÆ Purpose of the Contract + +In Market 2.0, contracts are used to: + +* Accept or reject deals +* Optionally implement additional business logic (e.g. FIL+ validation, payments, approvals) +* Return a DealID string if accepted + +The contract does **not** manage storage or retrieval itselfā€”that is handled by the SP. + +--- + +## āœ… Requirements + +A valid Market 2.0 contract must: + +1. Be deployed on a supported chain (e.g. Filecoin EVM, Hyperspace, etc) +2. Be whitelisted by the storage provider (via UI or admin tool) +3. Have its ABI uploaded +4. Expose a method that: + + * Accepts a single `bytes` input + * Returns a string (representing the DealID) + +--- + +## šŸ” Flow + +1. Client encodes parameters for your method +2. Client submits deal to Curio with: + + * Contract address + * Method name + * ABI-encoded parameters +3. Curio: + + * Loads ABI + * Packs the method call + * Calls `eth_call` + * Unpacks the return value + +If the method returns a string ā†’ deal is accepted. If empty string or call fails ā†’ deal is rejected. + +--- + +## šŸ§Ŗ Example Contract Method + +```solidity +function verifyDeal(bytes calldata params) external view returns (string memory) { + // decode params into your structure + // perform validation + // return deal ID if valid + return "deal-123"; +} +``` + +--- + +## šŸ“œ ABI Upload + +The SP must upload the ABI JSON for your contract when whitelisting it: + +* This enables Curio to find and call the method +* ABI must include the method name, inputs, and return types + +--- + +## šŸ” Client Responsibilities + +Clients must: + +* Choose a contract accepted by the SP +* Encode call parameters into `[]byte` +* Provide method name and contract address in the deal + +--- + +## šŸ§© Products and Contract Use + +Contracts are typically used from within a **product** (e.g. `ddov1`). The product defines: + +* Contract address +* Method name +* Encoded params (using ABI rules) + +This decouples contract logic from storage logic and keeps deals composable. + +--- + +## šŸš« Common Errors + +| Error | Cause | +| ------------------------------- | ---------------------------------------------- | +| `426 Deal rejected by contract` | Returned string is empty or `eth_call` fails | +| `ABI not found` | Contract not whitelisted or ABI missing | +| `Invalid method` | Method name not found in ABI | +| `Incorrect input format` | Method doesnā€™t accept single `bytes` parameter | + +--- + +## āœ… Checklist for Integration + +* [ ] Deploy contract on supported chain +* [ ] Expose a `function(bytes) returns (string)` method +* [ ] Whitelist contract via SP UI +* [ ] Upload ABI including the method +* [ ] Coordinate with clients on method + param encoding + +--- + +This guide enables market developers to plug in custom contract logic without requiring any changes to Curio or the storage pipeline. + +Welcome to programmable storage governance. diff --git a/documentation/en/market-2.0/products.md b/documentation/en/market-2.0/products.md new file mode 100644 index 000000000..00f277250 --- /dev/null +++ b/documentation/en/market-2.0/products.md @@ -0,0 +1,246 @@ +# Products & Extensibility Guide + +Market 2.0 introduces a fully extensible framework for storage deal configuration. This guide explains how products work, how new ones can be added, and how developers and providers can safely evolve without changing core Curio logic. + +--- + +## šŸ§© What Is a Product? + +A **product** is a named section of a deal that adds optional logic or configuration. Each product defines one or more aspects of the deal lifecycle. + +Examples: + +* `ddov1` ā€“ controls how data is onboarded and what contract governs it +* `aclv1` *(future)* ā€“ may define access control permissions +* `retrievalv1` *(future)* ā€“ may define retrieval conditions or SLA pricing + +Each product is a top-level field in the `products` object in a deal: + +```json +"products": { + "ddo_v1": { ... }, + "aclv1": { ... } +} +``` + +--- + +## šŸ›  Product Responsibilities + +A product may: + +* Validate a deal before acceptance +* Provide smart contract call details +* Affect retrieval behavior (e.g. IPNI, ACLs) +* Receive notifications (e.g. on sector sealing) + +A product **must not**: + +* Trigger storage actions directly (Curio handles onboarding) +* Conflict with other products +* Depend on runtime configuration (products are static per deal) + +--- + +## šŸ“ Product Structure + +All products are Go structs that implement the following interface-like behavior: + +* A `.Validate(*DB, *Config) (ErrorCode, error)` method +* Optional `.GetDealID()` logic if a contract call is needed +* Unique product name (`ProductNameDDOV1`, etc) + +Products are stored in JSON under the `products` field. + +--- + +## šŸ§Ŗ Example: `ddov1` + +The `ddov1` product includes: + +* Provider, client, and piece manager addresses +* Duration or allocation ID +* Smart contract call details: address, method, params +* Flags for indexing and IPNI +* Optional notification hooks + +Curio uses these fields to validate the deal, determine storage lifecycle, and optionally announce to IPNI. + +--- + +## šŸ›” ACLs and Retrieval Products (Future) + +Market 2.0 was designed to support retrieval-layer enforcement through: + +* ACL products (e.g., define who can retrieve what, when) +* Retrieval policy products (e.g., define pricing, terms) + +These will live alongside onboarding products like `ddov1`. + +--- + +## āœ… Design Philosophy + +* Each product handles one concern +* Multiple products can be included in one deal +* Future products won't require code changes to existing ones +* Extensibility is done via composition, not inheritance + +--- + +## šŸ“¦ Summary + +| Concept | Description | +| ------------ | -------------------------------------------------------- | +| Product | Modular block in a deal defining optional behavior | +| Validation | Each product validates its own logic | +| Contract | Products may define how to obtain deal ID | +| Future-proof | New products can be added without DB or protocol changes | + +Products are the core of Market 2.0's flexibilityā€”allowing new ideas to be layered in without disrupting existing workflows. + +# Write Your Own Product ā€“ Developer Guide + +This guide walks developers through creating a custom **product** for Market 2.0. Products add modular capabilities to dealsā€”ranging from storage control to retrieval logic, ACLs, SLAs, and beyond. + +--- + +Each product must: + +* Implement validation +* Optionally provide contract call instructions (if needed) +* Return its canonical product name + +--- + +## šŸ§± Structure of a Product + +Each product is a Go struct with a few key methods: + +```go +type MyProduct struct { + SomeField string `json:"some_field"` + // More fields... +} + +func (p *MyProduct) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { + // Check for required fields + // Enforce constraints + return Ok, nil +} + +func (p *MyProduct) ProductName() ProductName { + return "myproductv1" +} +``` + +--- + +## šŸ›  Adding a New Product (Step-by-Step) + +### 1. Define Struct in `types.go` + +Add a new `MyProduct` struct to the `Products` block: + +```go +type Products struct { + DDOV1 *DDOV1 `json:"ddo_v1"` + MyProduct *MyProduct `json:"myproduct_v1"` +} +``` + +### 2. Implement `.Validate()` + +Use `Validate()` to define how the product ensures the deal is valid. +You may: + +* Check required fields +* Enforce logic (e.g. if X is true, Y must also be set) +* Query DB if needed + +Return `ErrorCode` and reason for failure. + +### 3. Optionally: Contract Integration + +If your product relies on a contract, implement: + +```go +func (p *MyProduct) GetDealID(...) (string, ErrorCode, error) +``` + +This is how `ddov1` fetches DealID via contract call. + +### 4. Add to JSON Marshal/Unmarshal + +Nothing neededā€”`Products` already uses JSON tags. +Curio stores each product as a JSON field under `products` in DB. + +### 5. Update UI Toggle Support (Optional) + +Add a toggle entry in the admin panel: + +* `market_mk20_products` table +* Use your product name as key (`myproduct_v1`) +* Enable or disable per deployment + +### 6. Document via `/info` + +Update the markdown generator so your product shows up in `/market/mk20/info`. + +--- + +## šŸ§Ŗ Example Use Case: Retrieval Policy + +You might want to create `retrievalv1` with: + +```go +type RetrievalV1 struct { + PayPerByte bool `json:"pay_per_byte"` + MaxBandwidth int `json:"max_bandwidth_kbps"` + AllowedIPs []string `json:"allowed_ips"` +} +``` + +And enforce in `.Validate()`: + +```go +if p.PayPerByte && p.MaxBandwidth == 0 { + return ErrProductValidationFailed, xerrors.Errorf("bandwidth limit required for paid retrieval") +} +``` + +Later, your retrieval service can look up this product and apply pricing. + +--- + +## āœ… Guidelines + +| Rule | Description | +| ------------------ | ----------------------------------------- | +| āœ… Modular | Product should only affect its own logic | +| āœ… Optional | Products are opt-in per deal | +| āœ… Composable | Multiple products can exist in one deal | +| āŒ No Runtime State | Product logic is static and stateless | +| āŒ No Storage Logic | Curio handles onboarding, not the product | + +--- + +## šŸ”„ Deployment Considerations + +* Curio does not require a restart to recognize new products +* Products not enabled in DB will be rejected during validation +* Ensure all field names are `snake_case` in JSON + +--- + +## šŸ“¦ Summary + +Products are the extension mechanism of Market 2.0: + +* Validated independently +* Optional per deal +* Zero-conflict by design +* Fully extensible without schema or protocol changes + +Use them to inject new behaviors into Curio without touching the base system. + diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 3faa3c1c7..d10ab30ab 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -190,7 +190,6 @@ CREATE TABLE market_mk20_pipeline ( CREATE TABLE market_mk20_pipeline_waiting ( id TEXT PRIMARY KEY, waiting_for_data BOOLEAN DEFAULT FALSE, - started_put BOOLEAN DEFAULT FALSE, start_time TIMESTAMPTZ DEFAULT NULL ); diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 3c14bbdfa..20f0ef08f 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -66,11 +66,13 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { func Router(mdh *MK20DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "timeout reading request")) + mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "timeout request")) mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "timeout reading request")) mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "timeout reading request")) mux.Put("/data", mdh.mk20UploadDealData) mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "timeout reading request")) + mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "timeout reading request")) + mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "timeout reading request")) return mux } @@ -159,7 +161,7 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { // mk20supportedContracts retrieves supported contract addresses from the database and returns them as a JSON response. func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *http.Request) { var contracts mk20.SupportedContracts - err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM contracts") + err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM ddo_contracts") if err != nil { if errors.Is(err, pgx.ErrNoRows) { log.Errorw("no supported contracts found") @@ -222,6 +224,43 @@ func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Re // info serves the contents of the info file as a text/markdown response with HTTP 200 or returns an HTTP 500 on read/write failure. func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { + prods, srcs, err := mdh.dm.MK20Handler.Supported(r.Context()) + if err != nil { + log.Errorw("failed to get supported producers and sources", "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + var sb strings.Builder + + sb.WriteString(`

    Supported Products

    + +`) + + for name, enabled := range prods { + status := "Disabled" + if enabled { + status = "Enabled" + } + sb.WriteString(fmt.Sprintf("", name, status)) + } + sb.WriteString(`
    NameStatus
    %s%s
    `) + + sb.WriteString(`

    Supported Data Sources

    + +`) + + for name, enabled := range srcs { + status := "Disabled" + if enabled { + status = "Enabled" + } + sb.WriteString(fmt.Sprintf("", name, status)) + } + sb.WriteString(`
    NameStatus
    %s%s
    `) + + summaryHTML := sb.String() + var mdRenderer = goldmark.New( goldmark.WithExtensions( extension.GFM, @@ -243,48 +282,95 @@ func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to render markdown", http.StatusInternalServerError) return } - //if err := goldmark.Convert(infoMarkdown, &buf); err != nil { - // http.Error(w, "failed to render markdown", http.StatusInternalServerError) - // return - //} w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "text/html; charset=utf-8") - rendered := strings.ReplaceAll(buf.String(), "", `
    `) + renderedMarkdown := strings.ReplaceAll(buf.String(), "
    ", `
    `) + rendered := summaryHTML + renderedMarkdown htmlStr := fmt.Sprintf(` - - - - - Curio Deal Schema - - - + + +
    + %s +
    + + `, rendered) + + _, err = w.Write([]byte(htmlStr)) + if err != nil { + log.Errorw("failed to write info file", "err", err) + } +} + +func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Request) { + prods, _, err := mdh.dm.MK20Handler.Supported(r.Context()) + if err != nil { + log.Errorw("failed to get supported producers and sources", "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + var products mk20.SupportedProducts + for k, v := range prods { + if v { + products.Products = append(products.Products, k) } - table { - margin-top: 1rem; + } + resp, err := json.Marshal(products) + if err != nil { + log.Errorw("failed to marshal supported products", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(resp) + if err != nil { + log.Errorw("failed to write supported products", "err", err) + } +} + +func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http.Request) { + _, srcs, err := mdh.dm.MK20Handler.Supported(r.Context()) + if err != nil { + log.Errorw("failed to get supported producers and sources", "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + var sources mk20.SupportedDataSources + for k, v := range srcs { + if v { + sources.Sources = append(sources.Sources, k) } - - - -
    -%s -
    - -`, rendered) - - _, err := w.Write([]byte(htmlStr)) + } + resp, err := json.Marshal(sources) if err != nil { - log.Errorw("failed to write info file", "err", err) + log.Errorw("failed to marshal supported sources", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + _, err = w.Write(resp) + if err != nil { + log.Errorw("failed to write supported sources", "err", err) } } diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md index d4802b744..d121b28e9 100644 --- a/market/mk20/http/info.md +++ b/market/mk20/http/info.md @@ -76,6 +76,28 @@ Fetch markdown-formatted documentation that describes the supported deal schema, - `200 OK`: with markdown content of the info file - `500 Internal Server Error`: if file is not found or cannot be read +### šŸ§° GET /products + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A +Fetch json list of the supported products. + +- **Response**: + - `200 OK`: with json content + - `500 Internal Server Error`: if info cannot be read + +### šŸŒ GET /sources + +- **Content-Type**: N/A +- **Body**: N/A +- **Query Parameters**: N/A +Fetch json list of the supported data sources. + +- **Response**: + - `200 OK`: with json content + - `500 Internal Server Error`: if info cannot be read + ## Supported Deal Types This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission. @@ -224,6 +246,27 @@ SupportedContracts represents a collection of contract addresses supported by a |-------|------|-----|-------------| | Contracts | [[]string](https://pkg.go.dev/builtin#string) | json:"contracts" | Contracts represents a list of supported contract addresses in string format. | +### SupportedDataSources + +SupportedDataSources represents a collection of dats sources supported by the SP. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Sources | [[]string](https://pkg.go.dev/builtin#string) | json:"sources" | Contracts represents a list of supported contract addresses in string format. | + +### SupportedProducts + +SupportedProducts represents a collection of products supported by the SP. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Products | [[]string](https://pkg.go.dev/builtin#string) | json:"products" | Contracts represents a list of supported contract addresses in string format. | + +### TimeoutReader + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| + ### Constants for ErrorCode | Constant | Code | Description | diff --git a/market/mk20/http/test.html b/market/mk20/http/test.html deleted file mode 100644 index 44a71201e..000000000 --- a/market/mk20/http/test.html +++ /dev/null @@ -1,692 +0,0 @@ - - - - - - - Curio Deal Schema - - - - - -
    -

    Storage Market Interface

    -

    This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider.

    -

    šŸ“” MK20 HTTP API Overview

    -

    The MK20 storage market module provides a set of HTTP endpoints under /market/mk20 that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior.

    -

    Base URL

    -

    The base URL for all MK20 endpoints is:

    - 
    
-/market/mk20
-
-
    -

    šŸ”„ POST /store

    -

    Submit a new MK20 deal.

    -
      -
    • Content-Type: N/A
      • -
    • Body: N/A
      • -
    • Query Parameters: N/A
      • -
    • Response: -
        -
      • 200 OK: Deal accepted
        • -
      • Other HTTP codes indicate validation failure, rejection, or system errors
        • -
      -
      • -
    -

    šŸ§¾ GET /status?id=

    -

    Retrieve the current status of a deal.

    -
      -
    • Content-Type: application/json
      • -
    • Body: N/A
      • -
    • Query Parameters: -
        -
      • id: Deal identifier in ULID format
        • -
      -
      • -
    • Response: -
        -
      • 200 OK: JSON-encoded deal status information
        • -
      • 400 Bad Request: Missing or invalid ID
        • -
      • 500 Internal Server Error: If backend fails to respond
        • -
      -
      • -
    -

    šŸ“œ GET /contracts

    -
      -
    • -

      Content-Type: N/A

      -
      • -
    • -

      Body: N/A

      -
      • -
    • -

      Query Parameters: N/A
      - Return the list of contract addresses supported by the provider.

      -
      • -
    • -

      Response:

      - -
      • -
    -

    šŸ—‚ PUT /data?id=

    -

    Upload deal data after the deal has been accepted.

    -
      -
    • Content-Type: application/octet-stream
      • -
    • Body: Deal data bytes
      • -
    • Query Parameter:
      - -id: Deal identifier in ULID format
      • -
    • Headers: -
        -
      • Content-Length: must be deal's raw size
        • -
      -
      • -
    • Response: -
        -
      • 200 OK: if data is successfully streamed
        • -
      • 400, 413, or 415: on validation failures
        • -
      -
      • -
    -

    šŸ§  GET /info

    -
      -
    • -

      Content-Type: N/A

      -
      • -
    • -

      Body: N/A

      -
      • -
    • -

      Query Parameters: N/A
      - Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources.

      -
      • -
    • -

      Response:

      -
        -
      • 200 OK: with markdown content of the info file
        • -
      • 500 Internal Server Error: if file is not found or cannot be read
        • -
      -
      • -
    -

    Supported Deal Types

    -

    This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.

    -

    Deal

    -

    Deal represents a structure defining the details and components of a specific deal in the system.

    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Identifierulid.ULIDjson:"identifier"Identifier represents a unique identifier for the deal in UUID format.
    Datamk20.DataSourcejson:"data"Data represents the source of piece data and associated metadata.
    Productsmk20.Productsjson:"products"Products represents a collection of product-specific information associated with a deal
    -

    DataSource

    -

    DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin.

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    PieceCIDcid.Cidjson:"piece_cid"PieceCID represents the unique identifier for a piece of data, stored as a CID object.
    Sizeabi.PaddedPieceSizejson:"size"Size represents the size of the padded piece in the data source.
    Formatmk20.PieceDataFormatjson:"format"Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats.
    SourceHTTP*mk20.DataSourceHTTPjson:"source_http"SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.
    SourceAggregate*mk20.DataSourceAggregatejson:"source_aggregate"SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.
    SourceOffline*mk20.DataSourceOfflinejson:"source_offline"SourceOffline defines the data source for offline pieces, including raw size information.
    SourceHttpPut*mk20.DataSourceHttpPutjson:"source_httpput"SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import
    -

    Products

    - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    DDOV1*mk20.DDOV1json:"ddo_v1"DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)
    -

    DDOV1

    -

    DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details
    - for a DDO deal handling.

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Provideraddress.Addressjson:"provider"Provider specifies the address of the provider
    Clientaddress.Addressjson:"client"Client represents the address of the deal client
    PieceManageraddress.Addressjson:"piece_manager"Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs
    Durationabi.ChainEpochjson:"duration"Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400
    AllocationId*verifreg.AllocationIdjson:"allocation_id"AllocationId represents an aggregated allocation identifier for the deal.
    ContractAddressstringjson:"contract_address"ContractAddress specifies the address of the contract governing the deal
    ContractVerifyMethodstringjson:"contract_verify_method"ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract
    ContractVerifyMethodParams[]bytejson:"contract_verify_method_params"ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract
    NotificationAddressstringjson:"notification_address"NotificationAddress specifies the address to which notifications will be relayed to when sector is activated
    NotificationPayload[]bytejson:"notification_payload"NotificationPayload holds the notification data typically in a serialized byte array format.
    Indexingbooljson:"indexing"Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval
    AnnounceToIPNIbooljson:"announce_to_ipni"AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI).
    -

    DataSourceAggregate

    -

    DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces.

    - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Pieces[]mk20.DataSourcejson:"pieces"
    -

    DataSourceHTTP

    -

    DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    RawSizeuint64json:"rawsize"RawSize specifies the raw size of the data in bytes.
    URLs[]mk20.HttpUrljson:"urls"URLs lists the HTTP endpoints where the piece data can be fetched.
    -

    DataSourceHttpPut

    -

    DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted.

    - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    RawSizeuint64json:"raw_size"RawSize specifies the raw size of the data in bytes.
    -

    DataSourceOffline

    -

    DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece.

    - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    RawSizeuint64json:"raw_size"RawSize specifies the raw size of the data in bytes.
    -

    DealStatusResponse

    -

    DealStatusResponse represents the response of a deal's status, including its current state and an optional error message.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Statemk20.DealStatejson:"status"State indicates the current processing state of the deal as a DealState value.
    ErrorMsgstringjson:"error_msg"ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.
    -

    FormatAggregate

    -

    FormatAggregate represents the aggregated format for piece data, identified by its type.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Typemk20.AggregateTypejson:"type"Type specifies the type of aggregation for data pieces, represented by an AggregateType value.
    Sub[]mk20.PieceDataFormatjson:"sub"Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate
    -

    FormatBytes

    -

    FormatBytes defines the raw byte representation of data as a format.

    - - - - - - - - - -
    FieldTypeTagDescription
    -

    FormatCar

    -

    FormatCar represents the CAR (Content Addressable archive) format for piece data serialization.

    - - - - - - - - - -
    FieldTypeTagDescription
    -

    HttpUrl

    -

    HttpUrl represents an HTTP endpoint configuration for fetching piece data.

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    URLstringjson:"url"URL specifies the HTTP endpoint where the piece data can be fetched.
    Headershttp.Headerjson:"headers"HTTPHeaders represents the HTTP headers associated with the URL.
    Priorityuint64json:"priority"Priority indicates the order preference for using the URL in requests, with lower values having higher priority.
    Fallbackbooljson:"fallback"Fallback indicates whether this URL serves as a fallback option when other URLs fail.
    -

    PieceDataFormat

    -

    PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data.

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Car*mk20.FormatCarjson:"car"Car represents the optional CAR file format, including its metadata and versioning details.
    Aggregate*mk20.FormatAggregatejson:"aggregate"Aggregate holds a reference to the aggregated format of piece data.
    Raw*mk20.FormatBytesjson:"raw"Raw represents the raw format of the piece data, encapsulated as bytes.
    -

    SupportedContracts

    -

    SupportedContracts represents a collection of contract addresses supported by a system or application.

    - - - - - - - - - - - - - - - - - -
    FieldTypeTagDescription
    Contracts[]stringjson:"contracts"Contracts represents a list of supported contract addresses in string format.
    -

    Constants for ErrorCode

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ConstantCodeDescription
    Ok200Ok represents a successful operation with an HTTP status code of 200.
    ErrBadProposal400ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic.
    ErrMalformedDataSource430ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data.
    ErrUnsupportedDataSource422ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context.
    ErrUnsupportedProduct423ErrUnsupportedProduct indicates that the requested product is not supported by the provider.
    ErrProductNotEnabled424ErrProductNotEnabled indicates that the requested product is not enabled on the provider.
    ErrProductValidationFailed425ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data.
    ErrDealRejectedByMarket426ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules.
    ErrServiceMaintenance503ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503.
    ErrServiceOverloaded429ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment.
    ErrMarketNotEnabled440ErrMarketNotEnabled indicates that the market is not enabled for the requested operation.
    ErrDurationTooShort441ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold.
    -

    Constants for DealState

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    ConstantCodeDescription
    DealStateAccepted"accepted"DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system.
    DealStateProcessing"processing"DealStateProcessing represents the state of a deal currently being processed in the pipeline.
    DealStateSealing"sealing"DealStateSealing indicates that the deal is currently being sealed in the system.
    DealStateIndexing"indexing"DealStateIndexing represents the state where a deal is undergoing indexing in the system.
    DealStateFailed"failed"DealStateFailed indicates that the deal has failed due to an error during processing, sealing, or indexing.
    DealStateComplete"complete"DealStateComplete indicates that the deal has successfully completed all processing and is finalized in the system.
    - -
    - - \ No newline at end of file diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 866766324..d78ebd05d 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sync/atomic" "github.com/ethereum/go-ethereum/ethclient" logging "github.com/ipfs/go-log/v2" @@ -34,15 +35,16 @@ type MK20API interface { } type MK20 struct { - miners []address.Address - db *harmonydb.DB - api MK20API - ethClient *ethclient.Client - si paths.SectorIndex - cfg *config.CurioConfig - sm map[address.Address]abi.SectorSize - as *multictladdr.MultiAddressSelector - stor paths.StashStore + miners []address.Address + db *harmonydb.DB + api MK20API + ethClient *ethclient.Client + si paths.SectorIndex + cfg *config.CurioConfig + sm map[address.Address]abi.SectorSize + as *multictladdr.MultiAddressSelector + stor paths.StashStore + maxParallelUploads *atomic.Int32 } func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, stor paths.StashStore) (*MK20, error) { @@ -61,15 +63,16 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } return &MK20{ - miners: miners, - db: db, - api: mapi, - ethClient: ethClient, - si: si, - cfg: cfg, - sm: sm, - as: as, - stor: stor, + miners: miners, + db: db, + api: mapi, + ethClient: ethClient, + si: si, + cfg: cfg, + sm: sm, + as: as, + stor: stor, + maxParallelUploads: new(atomic.Int32), }, nil } diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index 772172ca4..de4e3a3c7 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -140,32 +140,52 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons ctx := context.Background() defer data.Close() - var waitingDeal []struct { - Started bool `db:"started_put"` - StartTime *time.Time `db:"start_time"` - } - - err := m.db.Select(ctx, &waitingDeal, `SELECT started_put, start_time from market_mk20_pipeline_waiting - WHERE waiting_for_data = TRUE AND id = $1`, id.String()) + var ( + idStr string + updated bool + ) + + err := m.db.QueryRow(ctx, `WITH check_row AS ( + SELECT id FROM market_mk20_pipeline_waiting + WHERE id = $1 AND waiting_for_data = TRUE + ), + try_update AS ( + UPDATE market_mk20_pipeline_waiting + SET start_time = NOW() + WHERE id IN (SELECT id FROM check_row) + AND ( + start_time IS NULL + OR start_time < NOW() - ($2 * INTERVAL '1 second') + ) + RETURNING id + ) + SELECT + check_row.id, + try_update.id IS NOT NULL AS updated + FROM check_row + LEFT JOIN try_update ON check_row.id = try_update.id;`, + id.String(), m.cfg.HTTP.ReadTimeout.Seconds()).Scan(&idStr, &updated) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + http.Error(w, "", http.StatusNotFound) + return + } log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) http.Error(w, "", http.StatusInternalServerError) return } - if len(waitingDeal) == 0 { - http.Error(w, "", http.StatusNotFound) + if idStr != id.String() { + log.Errorw("deal id mismatch", "deal", id.String(), "db", idStr) + http.Error(w, "", http.StatusInternalServerError) + return } - if waitingDeal[0].Started { - if waitingDeal[0].StartTime.Add(m.cfg.HTTP.ReadTimeout).Before(time.Now()) { - http.Error(w, "another /PUT request is in progress for this deal", http.StatusConflict) - } + if !updated { + http.Error(w, "", http.StatusConflict) } - // TODO: Rethink how to ensure only 1 process per deal for /PUT - deal, err := DealFromDB(ctx, m.db, id) if err != nil { log.Errorw("failed to get deal from db", "deal", id.String(), "err", err) @@ -196,6 +216,7 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons failed := true defer func() { + m.maxParallelUploads.Add(-1) if failed { _, err = m.db.Exec(ctx, `UPDATE market_mk20_pipeline_waiting SET started_put = FALSE, start_time = NULL WHERE id = $1`, id.String()) if err != nil { @@ -204,14 +225,22 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons } }() + if m.maxParallelUploads.Load() >= int32(m.cfg.Market.StorageMarketConfig.MK20.MaxParallelUploads) { + http.Error(w, "Too many parallel uploads", http.StatusTooManyRequests) + return + } + + m.maxParallelUploads.Add(1) + cp := new(commp.Calc) + reader := NewTimeoutReader(data, time.Second*5) // Function to write data into StashStore and calculate commP writeFunc := func(f *os.File) error { - limitedReader := io.LimitReader(data, int64(rawSize+1)) // +1 to detect exceeding the limit + limitedReader := io.LimitReader(reader, int64(rawSize+1)) // +1 to detect exceeding the limit wr := io.MultiWriter(f, cp) - size, err := io.Copy(wr, limitedReader) + size, err := io.CopyBuffer(wr, limitedReader, make([]byte, 4<<20)) if err != nil { return fmt.Errorf("failed to read and write piece data: %w", err) } @@ -381,3 +410,74 @@ func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.Respons w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("OK")) } + +func (m *MK20) Supported(ctx context.Context) (map[string]bool, map[string]bool, error) { + var products []struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` + } + err := m.db.Select(ctx, &products, `SELECT name, enabled FROM market_mk20_products`) + if err != nil { + return nil, nil, err + } + + productsMap := make(map[string]bool) + + for _, product := range products { + productsMap[product.Name] = product.Enabled + } + + var sources []struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` + } + err = m.db.Select(ctx, &sources, `SELECT name, enabled FROM market_mk20_data_source`) + if err != nil { + return nil, nil, err + } + sourcesMap := make(map[string]bool) + for _, source := range sources { + sourcesMap[source.Name] = source.Enabled + } + return productsMap, sourcesMap, nil +} + +type TimeoutReader struct { + r io.Reader + timeout time.Duration +} + +func NewTimeoutReader(r io.Reader, timeout time.Duration) *TimeoutReader { + return &TimeoutReader{ + r: r, + timeout: timeout, + } +} + +func (t *TimeoutReader) Read(p []byte) (int, error) { + deadline := time.Now().Add(t.timeout) + for { + // Attempt to read + n, err := t.r.Read(p) + + if err != nil { + return n, err + } + + if n > 0 { + // Successfully read some data; reset the deadline + deadline = time.Now().Add(t.timeout) + + // Otherwise return bytes read and no error + return n, err + } + + // Timeout: If we hit the deadline without making progress, return a timeout error + if time.Now().After(deadline) { + return 0, fmt.Errorf("upload timeout: no progress (duration: %f Seconds)", t.timeout.Seconds()) + } + + // Avoid tight loop by adding a tiny sleep + time.Sleep(100 * time.Millisecond) // Small pause to avoid busy-waiting + } +} diff --git a/market/mk20/mk20gen/gen.go b/market/mk20/mk20gen/gen.go index fce1c85a1..374b8808e 100644 --- a/market/mk20/mk20gen/gen.go +++ b/market/mk20/mk20gen/gen.go @@ -272,6 +272,24 @@ func writeOutput(path string) { buf.WriteString(" - `200 OK`: with markdown content of the info file\n") buf.WriteString(" - `500 Internal Server Error`: if file is not found or cannot be read\n\n") + buf.WriteString("### šŸ§° GET /products\n\n") + buf.WriteString("- **Content-Type**: N/A\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**: N/A\n") + buf.WriteString("Fetch json list of the supported products.\n\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: with json content\n") + buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") + + buf.WriteString("### šŸŒ GET /sources\n\n") + buf.WriteString("- **Content-Type**: N/A\n") + buf.WriteString("- **Body**: N/A\n") + buf.WriteString("- **Query Parameters**: N/A\n") + buf.WriteString("Fetch json list of the supported data sources.\n\n") + buf.WriteString("- **Response**:\n") + buf.WriteString(" - `200 OK`: with json content\n") + buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") + buf.WriteString("## Supported Deal Types\n\n") buf.WriteString("This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.\n\n") diff --git a/market/mk20/utils.go b/market/mk20/utils.go index e968620b1..28e7049bd 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -635,3 +635,15 @@ func IsProductEnabled(db *harmonydb.DB, name ProductName) (ErrorCode, error) { } return Ok, nil } + +// SupportedProducts represents a collection of products supported by the SP. +type SupportedProducts struct { + // Contracts represents a list of supported contract addresses in string format. + Products []string `json:"products"` +} + +// SupportedDataSources represents a collection of dats sources supported by the SP. +type SupportedDataSources struct { + // Contracts represents a list of supported contract addresses in string format. + Sources []string `json:"sources"` +} diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index 757d21b84..1c66b43a6 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/samber/lo" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -231,7 +232,7 @@ func (a *WebRPC) IPNISummary(ctx context.Context) ([]*IPNI, error) { return nil, fmt.Errorf("failed to fetch IPNI configuration: %w", err) } - for _, service := range services { + for _, service := range lo.Uniq(services) { for _, d := range summary { url := service + "/providers/" + d.PeerID resp, err := http.Get(url) diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index 768a328b8..a805493c6 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -9,6 +9,8 @@ import ( "strings" "time" + eabi "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" @@ -561,3 +563,246 @@ func (a *WebRPC) MK20BulkRemoveFailedMarketPipelines(ctx context.Context, taskTy return nil } + +func (a *WebRPC) AddMarketContract(ctx context.Context, contract, abiString string) error { + if contract == "" { + return fmt.Errorf("empty contract") + } + if abiString == "" { + return fmt.Errorf("empty abi") + } + + if !strings.HasPrefix(contract, "0x") { + return fmt.Errorf("contract must start with 0x") + } + + if !common.IsHexAddress(contract) { + return fmt.Errorf("invalid contract address") + } + + ethabi, err := eabi.JSON(strings.NewReader(abiString)) + if err != nil { + return fmt.Errorf("invalid abi: %w", err) + } + + if ethabi.Methods == nil || len(ethabi.Methods) == 0 { + return fmt.Errorf("invalid abi: no methods") + } + + n, err := a.deps.DB.Exec(ctx, `INSERT INTO ddo_contracts (address, abi) VALUES ($1, $2) ON CONFLICT (address) DO NOTHING`, contract, abiString) + if err != nil { + return xerrors.Errorf("failed to add contract: %w", err) + } + if n == 0 { + return fmt.Errorf("contract already exists") + } + return nil +} + +func (a *WebRPC) UpdateMarketContract(ctx context.Context, contract, abiString string) error { + if contract == "" { + return fmt.Errorf("empty contract") + } + + if abiString == "" { + return fmt.Errorf("empty abi") + } + + if !strings.HasPrefix(contract, "0x") { + return fmt.Errorf("contract must start with 0x") + } + + if !common.IsHexAddress(contract) { + return fmt.Errorf("invalid contract address") + } + + ethabi, err := eabi.JSON(strings.NewReader(abiString)) + if err != nil { + return fmt.Errorf("invalid abi: %w", err) + } + + if ethabi.Methods == nil || len(ethabi.Methods) == 0 { + return fmt.Errorf("invalid abi: no methods") + } + + // Check if contract exists in DB + var count int + err = a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM ddo_contracts WHERE address = $1`, contract).Scan(&count) + if err != nil { + return xerrors.Errorf("failed to check contract: %w", err) + } + if count == 0 { + return fmt.Errorf("contract does not exist") + } + + n, err := a.deps.DB.Exec(ctx, `UPDATE ddo_contracts SET abi = $2 WHERE address = $1`, contract, abiString) + if err != nil { + return xerrors.Errorf("failed to update contract ABI: %w", err) + } + + if n == 0 { + return fmt.Errorf("failed to update the contract ABI") + } + + return nil +} + +func (a *WebRPC) RemoveMarketContract(ctx context.Context, contract string) error { + if contract == "" { + return fmt.Errorf("empty contract") + } + if !strings.HasPrefix(contract, "0x") { + return fmt.Errorf("contract must start with 0x") + } + _, err := a.deps.DB.Exec(ctx, `DELETE FROM ddo_contracts WHERE address = $1`, contract) + if err != nil { + return xerrors.Errorf("failed to remove contract: %w", err) + } + return nil +} + +func (a *WebRPC) ListMarketContracts(ctx context.Context) (map[string]string, error) { + var contracts []struct { + Address string `db:"address"` + Abi string `db:"abi"` + } + err := a.deps.DB.Select(ctx, &contracts, `SELECT address, abi FROM ddo_contracts`) + if err != nil { + return nil, xerrors.Errorf("failed to get contracts from DB: %w", err) + } + + contractMap := make(map[string]string) + for _, contract := range contracts { + contractMap[contract.Address] = contract.Abi + } + + return contractMap, nil +} + +func (a *WebRPC) EnableProduct(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("empty product name") + } + + // Check if product exists in market_mk20_products + var count int + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM market_mk20_products WHERE name = $1`, name).Scan(&count) + if err != nil { + return xerrors.Errorf("failed to check product: %w", err) + } + if count == 0 { + return fmt.Errorf("product does not exist") + } + n, err := a.deps.DB.Exec(ctx, `UPDATE market_mk20_products SET enabled = true WHERE name = $1`, name) + if err != nil { + return xerrors.Errorf("failed to enable product: %w", err) + } + if n == 0 { + return fmt.Errorf("failed to enable the product") + } + return nil +} + +func (a *WebRPC) DisableProduct(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("empty product name") + } + + // Check if product exists in market_mk20_products + var count int + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM market_mk20_products WHERE name = $1`, name).Scan(&count) + if err != nil { + return xerrors.Errorf("failed to check product: %w", err) + } + if count == 0 { + return fmt.Errorf("product does not exist") + } + n, err := a.deps.DB.Exec(ctx, `UPDATE market_mk20_products SET enabled = false WHERE name = $1`, name) + if err != nil { + return xerrors.Errorf("failed to disable product: %w", err) + } + if n == 0 { + return fmt.Errorf("failed to disable the product") + } + return nil +} + +func (a *WebRPC) ListProducts(ctx context.Context) (map[string]bool, error) { + var products []struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` + } + err := a.deps.DB.Select(ctx, &products, `SELECT name, enabled FROM market_mk20_products`) + if err != nil { + return nil, xerrors.Errorf("failed to get products from DB: %w", err) + } + productMap := make(map[string]bool) + for _, product := range products { + productMap[product.Name] = product.Enabled + } + return productMap, nil +} + +func (a *WebRPC) EnableDataSource(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("empty data source name") + } + + // check if datasource exists in market_mk20_data_source + var count int + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM market_mk20_data_source WHERE name = $1`, name).Scan(&count) + if err != nil { + return xerrors.Errorf("failed to check datasource: %w", err) + } + if count == 0 { + return fmt.Errorf("datasource does not exist") + } + n, err := a.deps.DB.Exec(ctx, `UPDATE market_mk20_data_source SET enabled = true WHERE name = $1`, name) + if err != nil { + return xerrors.Errorf("failed to enable datasource: %w", err) + } + if n == 0 { + return fmt.Errorf("failed to enable the datasource") + } + return nil +} + +func (a *WebRPC) DisableDataSource(ctx context.Context, name string) error { + if name == "" { + return fmt.Errorf("empty data source name") + } + // check if datasource exists in market_mk20_data_source + var count int + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM market_mk20_data_source WHERE name = $1`, name).Scan(&count) + if err != nil { + return xerrors.Errorf("failed to check datasource: %w", err) + } + if count == 0 { + return fmt.Errorf("datasource does not exist") + } + n, err := a.deps.DB.Exec(ctx, `UPDATE market_mk20_data_source SET enabled = false WHERE name = $1`, name) + if err != nil { + return xerrors.Errorf("failed to disable datasource: %w", err) + } + if n == 0 { + return fmt.Errorf("failed to disable the datasource") + } + return nil +} + +func (a *WebRPC) ListDataSources(ctx context.Context) (map[string]bool, error) { + var datasources []struct { + Name string `db:"name"` + Enabled bool `db:"enabled"` + } + err := a.deps.DB.Select(ctx, &datasources, `SELECT name, enabled FROM market_mk20_data_source`) + if err != nil { + return nil, xerrors.Errorf("failed to get datasources from DB: %w", err) + } + + datasourceMap := make(map[string]bool) + for _, datasource := range datasources { + datasourceMap[datasource.Name] = datasource.Enabled + } + return datasourceMap, nil +} diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 137baf2bb..87b0ff77c 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -11,6 +11,11 @@ class DealDetails extends LitElement { this.loaddata(); } + createRenderRoot() { + return this; // Render into light DOM instead of shadow DOM + } + + async loaddata() { try { const params = new URLSearchParams(window.location.search); @@ -23,7 +28,6 @@ class DealDetails extends LitElement { } render() { - console.log(this.data); if (!this.data) return html`

    No data.

    `; const { identifier, data, products, error } = this.data.deal; @@ -42,7 +46,6 @@ class DealDetails extends LitElement { Error PieceCID${data?.piece_cid['/']} PieceSize${data?.piece_size} - Error

    Piece Format

    @@ -51,8 +54,8 @@ class DealDetails extends LitElement {

    Data Source

    - - + + ${this.renderDataSource(data)} @@ -68,7 +71,7 @@ class DealDetails extends LitElement { if (data.source_http) { return html` - + ` @@ -76,7 +79,7 @@ class DealDetails extends LitElement { if (data.source_aggregate) { return html` - + ` @@ -84,7 +87,7 @@ class DealDetails extends LitElement { if (data.source_offline) { return html` - + ` @@ -92,7 +95,7 @@ class DealDetails extends LitElement { if (data.source_httpput) { return html` - + ` @@ -124,7 +127,7 @@ class DealDetails extends LitElement { return html`
    NameDetailsNameDetails
    HTTPHTTP ${data?.source_http ? this.renderSourceHTTP(data.source_http) : ''}
    AggregateAggregate ${data?.source_aggregate ? this.renderSourceAggregate(data.source_aggregate) : ''}
    OfflineOffline ${data?.source_offline ? this.renderSourceOffline(data.source_offline) : ''}
    HTTP PutHTTP Put ${data?.source_httpput ? this.renderSourceHttpPut(data.source_httpput) : ''}
    - ${src.urls ? this.renderUrls(src.urls) : ''} +
    Raw Size${src.rawsize}
    ${src.urls ? this.renderUrls(src.urls) : ''}
    `; } @@ -160,26 +163,29 @@ class DealDetails extends LitElement { renderSourceAggregate(src) { return html` -
    - [Aggregate Details] -
    - ${src.pieces.map((piece, i) => html` -
    - - Piece ${i + 1} - - - - - - -
    PieceCID${piece.piece_cid['/']}
    Size${piece.piece_size}
    ${this.renderPieceFormat(piece.format)}
    ${this.renderDataSource(piece)}
    - - +
    + [Aggregate Details] +
    + ${src.pieces.map((piece, i) => html` +
    +

    + +

    +
    +
    +
      +
    • PieceCID: ${piece.piece_cid['/']} Size: ${piece.piece_size}
      • +
    • ${this.renderPieceFormat(piece.format)}
      • +
    • ${this.renderDataSource(piece)}
      • +
    +
    +
    - `)} -
    -
    + `)} +
    +
    `; } diff --git a/web/static/pages/mk20-deal/index.html b/web/static/pages/mk20-deal/index.html index f05a776bf..6a8cefe55 100644 --- a/web/static/pages/mk20-deal/index.html +++ b/web/static/pages/mk20-deal/index.html @@ -6,6 +6,7 @@ + diff --git a/web/static/pages/mk20/deal-search.mjs b/web/static/pages/mk20/deal-search.mjs new file mode 100644 index 000000000..42f6430e0 --- /dev/null +++ b/web/static/pages/mk20/deal-search.mjs @@ -0,0 +1,68 @@ +import { html, css, LitElement } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; + +class DealSearch extends LitElement { + static properties = { + searchTerm: { type: String }, + }; + + constructor() { + super(); + this.searchTerm = ''; + } + + handleInput(event) { + this.searchTerm = event.target.value; + } + + handleSearch() { + if (this.searchTerm.trim() !== '') { + window.location.href = `/pages/mk20-deal/?id=${encodeURIComponent(this.searchTerm.trim())}`; + } + // If searchTerm is empty, do nothing + } + + render() { + return html` + +
    + + +
    + `; + } + + static styles = css` + .search-container { + display: grid; + grid-template-columns: 1fr max-content; + grid-column-gap: 0.75rem; + margin-bottom: 1rem; + } + + .btn { + padding: 0.4rem 1rem; + border: none; + border-radius: 0; + background-color: var(--color-form-default); + color: var(--color-text-primary); + + &:hover, &:focus, &:focus-visible { + background-color: var(--color-form-default-pressed); + color: var(--color-text-secondary); + } + } + `; +} + +customElements.define('deal-search', DealSearch); diff --git a/web/static/pages/mk20/index.html b/web/static/pages/mk20/index.html index 9ae4bf8f1..0a1c79891 100644 --- a/web/static/pages/mk20/index.html +++ b/web/static/pages/mk20/index.html @@ -6,6 +6,8 @@ + + @@ -14,12 +16,30 @@

    Storage Deals

    +
    +
    + +
    +
    +
    +
    +
    +

    Settings

    +
    +
    +
    + +
    +
    +
    +
    +
    diff --git a/web/static/pages/mk20/settings.mjs b/web/static/pages/mk20/settings.mjs new file mode 100644 index 000000000..5f0b2cab5 --- /dev/null +++ b/web/static/pages/mk20/settings.mjs @@ -0,0 +1,297 @@ +import { html, css, LitElement } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; + +/** + * A custom Web Component for managing products, data sources, and market contracts. + * Extends the LitElement class to leverage the Lit library for efficient rendering. + */ +class MarketManager extends LitElement { + static properties = { + products: { type: Array }, + dataSources: { type: Array }, + contracts: { type: Array }, + selectedContract: { type: Object }, + }; + + constructor() { + super(); + this.products = []; + this.dataSources = []; + this.contracts = []; + this.selectedContract = null; // For modal + this.loadAllData(); + } + + async loadAllData() { + try { + const productsResult = await RPCCall('ListProducts', []); + this.products = Array.isArray(productsResult) + ? productsResult + : Object.entries(productsResult).map(([name, enabled]) => ({ name, enabled })); + + const dataSourcesResult = await RPCCall('ListDataSources', []); + this.dataSources = Array.isArray(dataSourcesResult) + ? dataSourcesResult + : Object.entries(dataSourcesResult).map(([name, enabled]) => ({ name, enabled })); + + const contractsResult = await RPCCall('ListMarketContracts', []); + this.contracts = Array.isArray(contractsResult) + ? contractsResult + : Object.entries(contractsResult).map(([address, abi]) => ({ address, abi })); + + this.requestUpdate(); + } catch (err) { + console.error('Failed to load data:', err); + this.products = []; + this.dataSources = []; + this.contracts = []; + } + } + + async toggleProductState(product) { + const confirmation = confirm( + `Are you sure you want to ${product.enabled ? 'disable' : 'enable'} the product "${product.name}"?` + ); + + if (!confirmation) return; + + try { + if (product.enabled) { + await RPCCall('DisableProduct', [product.name]); + } else { + await RPCCall('EnableProduct', [product.name]); + } + this.loadAllData(); // Refresh after toggling + } catch (err) { + console.error('Failed to toggle product state:', err); + } + } + + async toggleDataSourceState(dataSource) { + const confirmation = confirm( + `Are you sure you want to ${dataSource.enabled ? 'disable' : 'enable'} the data source "${dataSource.name}"?` + ); + + if (!confirmation) return; + + try { + if (dataSource.enabled) { + await RPCCall('DisableDataSource', [dataSource.name]); + } else { + await RPCCall('EnableDataSource', [dataSource.name]); + } + this.loadAllData(); // Refresh after toggling + } catch (err) { + console.error('Failed to toggle data source state:', err); + } + } + + openContractModal(contract) { + this.selectedContract = { ...contract }; + this.updateComplete.then(() => { + const modal = this.shadowRoot.querySelector('#contract-modal'); + if (modal && typeof modal.showModal === 'function') { + modal.showModal(); + } + }); + } + + async removeContract(contract) { + if (!confirm(`Are you sure you want to remove contract ${contract.address}?`)) return; + + try { + await RPCCall('RemoveMarketContract', [contract.address]); + await this.loadAllData(); + } catch (err) { + console.error('Failed to remove contract:', err); + alert(`Failed to remove contract: ${err.message}`); + } + } + + + async saveContractChanges() { + try { + const { address, abi } = this.selectedContract; + + if (!address || !abi) { + alert("Contract address and ABI are required."); + return; + } + + const method = this.contracts.find(c => c.address === address) + ? 'UpdateMarketContract' + : 'AddMarketContract'; + + await RPCCall(method, [address, abi]); + + this.loadAllData(); + this.closeModal(); + } catch (err) { + console.error('Failed to save contract changes:', err); + alert(`Failed to save contract: ${err.message}`); + } + } + + + closeModal() { + this.selectedContract = null; + const modal = this.shadowRoot.querySelector('#contract-modal'); + if (modal) modal.close(); + } + + render() { + return html` + + + +
    + + + + + + +
    +

    Products

    + + + + + + + + + + ${this.products?.map( + (product) => html` + + + + + + ` + )} + +
    NameEnabledAction
    ${product.name}${product.enabled ? 'Yes' : 'No'} + +
    +     		+

    Data Sources

    + + + + + + + + + + ${this.dataSources?.map( + (source) => html` + + + + + + ` + )} + +
    NameEnabledAction
    ${source.name}${source.enabled ? 'Yes' : 'No'} + +
    +     		+

    Contracts + +

    + + + + + + + + + ${this.contracts?.map( + (contract) => html` + + + + + ` + )} + +
    AddressActions
    ${contract.address} +
    + + +
    +
    +
    + ${this.renderContractModal()} +
    + `; + } + + renderContractModal() { + if (!this.selectedContract) return null; + + return html` + +

    + ${this.contracts.some(c => c.address === this.selectedContract.address) + ? html`Edit Contract: ${this.selectedContract.address}` + : html`Add New Contract`} +

    + +
    + + this.selectedContract.address = e.target.value} + placeholder="0x..." + /> +
    + +
    + + +
    + +
    + + +
    +     + `; + } +} + +customElements.define('market-manager', MarketManager); \ No newline at end of file From 0b7a76f71e9110ee4e9fb160b69fad42bbc27e06 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 11 Jun 2025 16:32:35 +0400 Subject: [PATCH 13/55] chunk upload --- alertmanager/alerts.go | 2 +- alertmanager/plugin/slack_webhook.go | 66 +++- cmd/curio/tasks/tasks.go | 3 +- cmd/sptool/toolbox_deal_client.go | 250 ++++++++++++ deps/config/doc_gen.go | 16 +- deps/config/types.go | 14 +- .../piece-server/sample/mk20-random-deal.sh | 48 ++- .../default-curio-configuration.md | 14 +- documentation/en/curio-cli/sptool.md | 16 + go.mod | 2 +- go.sum | 4 +- .../harmonydb/sql/20250505-market_mk20.sql | 16 +- lib/paths/http_handler.go | 1 + market/mk20/http/http.go | 171 ++++++--- market/mk20/http/info.md | 22 +- market/mk20/mk20.go | 13 +- market/mk20/mk20_upload.go | 362 ++++++++++++++++++ market/mk20/mk20_utils.go | 289 -------------- market/mk20/types.go | 4 +- market/mk20/utils.go | 24 ++ pdp/handlers_upload.go | 3 +- tasks/piece/task_aggregate_chunks.go | 341 +++++++++++++++++ web/api/webrpc/market_20.go | 4 +- 23 files changed, 1295 insertions(+), 390 deletions(-) create mode 100644 market/mk20/mk20_upload.go create mode 100644 tasks/piece/task_aggregate_chunks.go diff --git a/alertmanager/alerts.go b/alertmanager/alerts.go index fafe65408..65f12df6e 100644 --- a/alertmanager/alerts.go +++ b/alertmanager/alerts.go @@ -731,7 +731,7 @@ func missingSectorCheck(al *alerts) { SectorID int64 `db:"sector_num"` } - err := al.db.Select(al.ctx, §ors, `SELECT miner_id, sector_num FROM sector_location WHERE sector_filetype = 2 GROUP BY miner_id, sector_num ORDER BY miner_id, sector_num`) + err := al.db.Select(al.ctx, §ors, `SELECT miner_id, sector_num FROM sector_location WHERE sector_filetype = ANY(ARRAY[2,8]) GROUP BY miner_id, sector_num ORDER BY miner_id, sector_num`) if err != nil { al.alertMap[Name].err = xerrors.Errorf("getting sealed sectors from database: %w", err) return diff --git a/alertmanager/plugin/slack_webhook.go b/alertmanager/plugin/slack_webhook.go index 118e92d4c..b524a036f 100644 --- a/alertmanager/plugin/slack_webhook.go +++ b/alertmanager/plugin/slack_webhook.go @@ -65,29 +65,60 @@ func (s *SlackWebhook) SendAlert(data *AlertPayload) error { // Iterate through the map to construct the remaining blocks for key, value := range data.Details { - // Split value into sentences by period followed by space + // Split value into sentences by period followed by space. sentences := strings.Split(value.(string), ". ") - formattedValue := fmt.Sprintf("ā€¢ *%s*\n", key) - // Add a bullet point before each trimmed sentence + // Add the key as the header for each block. + baseFormattedValue := fmt.Sprintf("ā€¢ *%s*\n", key) + currentFormattedValue := baseFormattedValue + + // Keep track of the character limit (3000) when adding sentences. for _, sentence := range sentences { - trimmedSentence := strings.TrimSpace(sentence) // Trim leading and trailing spaces + trimmedSentence := strings.TrimSpace(sentence) // Trim leading and trailing spaces. if trimmedSentence != "" { - formattedValue += fmt.Sprintf("ā€¢ %s.\n", trimmedSentence) // Add period back and newline + // Add a bullet point and sentence, restoring the period and newline. + newSection := fmt.Sprintf("ā€¢ %s.\n", trimmedSentence) + + // Check if adding this section exceeds the 3000-character limit. + if len(currentFormattedValue)+len(newSection) > 3000 { + // If limit exceeds, add the currentFormattedValue block to payload and start a new block. + payload.Blocks = append(payload.Blocks, + Block{ + Type: "section", + Text: &TextBlock{ + Type: "mrkdwn", + Text: currentFormattedValue, + }, + }, + Block{ + Type: "divider", + }, + ) + + // Start a new formatted value with the baseFormattedValue. + currentFormattedValue = baseFormattedValue + } + + // Append the newSection to the currentFormattedValue. + currentFormattedValue += newSection } } - payload.Blocks = append(payload.Blocks, - Block{ - Type: "section", - Text: &TextBlock{ - Type: "mrkdwn", - Text: formattedValue, + + // Add the last block if it contains any content. + if currentFormattedValue != baseFormattedValue { + payload.Blocks = append(payload.Blocks, + Block{ + Type: "section", + Text: &TextBlock{ + Type: "mrkdwn", + Text: currentFormattedValue, + }, }, - }, - Block{ - Type: "divider", - }, - ) + Block{ + Type: "divider", + }, + ) + } } // Marshal the payload to JSON @@ -163,7 +194,8 @@ func (s *SlackWebhook) SendAlert(data *AlertPayload) error { } }) if err != nil { - return fmt.Errorf("after %d retries,last error: %w", iter, err) + log.Errorw("Slack Webhook payload:", string(jsonData)) + return fmt.Errorf("after %d retries,last error: %w, %s", iter, err, string(jsonData)) } return nil } diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index e8ef56e40..ddcb03ef9 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -225,7 +225,8 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan return nil, err } cleanupPieceTask := piece2.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0) - activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask) + aggregateChunksTask := piece2.NewAggregateChunksTask(db, lstor, stor) + activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask, aggregateChunksTask) } } diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index af0d82e5c..29318f02c 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -33,7 +33,9 @@ import ( "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multibase" + "github.com/oklog/ulid" "github.com/urfave/cli/v2" + "golang.org/x/exp/mmap" "golang.org/x/term" "golang.org/x/xerrors" @@ -1579,6 +1581,7 @@ var mk20Clientcmd = &cli.Command{ initCmd, mk20DealCmd, mk20ClientMakeAggregateCmd, + mk20ClientUploadCmd, }, } @@ -1984,3 +1987,250 @@ var mk20ClientMakeAggregateCmd = &cli.Command{ return nil }, } + +var mk20ClientUploadCmd = &cli.Command{ + Name: "upload", + Usage: "Upload a file to the storage provider", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "provider", + Usage: "storage provider on-chain address", + Required: true, + }, + &cli.StringFlag{ + Name: "deal", + Usage: "deal id to upload to", + Required: true, + }, + &cli.StringFlag{ + Name: "chunk-size", + Usage: "chunk size to be used for the upload", + Value: "4 MiB", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 1 { + return xerrors.Errorf("must provide a single file to upload") + } + file := cctx.Args().First() + log.Debugw("uploading file", "file", file) + ctx := cctx.Context + + chunkSizeStr := cctx.String("chunk-size") + chunkSizem, err := humanize.ParseBytes(chunkSizeStr) + if err != nil { + return xerrors.Errorf("parsing chunk size: %w", err) + } + + if chunkSizem == 0 { + return xerrors.Errorf("invalid chunk size: %s", chunkSizeStr) + } + + // Verify chunk size is power of 2 + if chunkSizem&(chunkSizem-1) != 0 { + return xerrors.Errorf("chunk size must be power of 2") + } + + chunkSize := int64(chunkSizem) + + dealid, err := ulid.Parse(cctx.String("deal")) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + + maddr, err := address.NewFromString(cctx.String("provider")) + if err != nil { + return err + } + + f, err := os.OpenFile(file, os.O_RDONLY, 0644) + if err != nil { + return xerrors.Errorf("opening file: %w", err) + } + + stat, err := f.Stat() + if err != nil { + return xerrors.Errorf("stat file: %w", err) + } + + size := stat.Size() + if size == 0 { + return xerrors.Errorf("file size is 0") + } + + if size < chunkSize { + chunkSize = size + } + + // Calculate the number of chunks + numChunks := int((size + chunkSize - 1) / chunkSize) + + f.Close() + + api, closer, err := lcli.GetGatewayAPIV1(cctx) + if err != nil { + return fmt.Errorf("cant setup gateway connection: %w", err) + } + defer closer() + + minfo, err := api.StateMinerInfo(ctx, maddr, chain_types.EmptyTSK) + if err != nil { + return err + } + if minfo.PeerId == nil { + return xerrors.Errorf("storage provider %s has no peer ID set on-chain", maddr) + } + + var maddrs []multiaddr.Multiaddr + for _, mma := range minfo.Multiaddrs { + ma, err := multiaddr.NewMultiaddrBytes(mma) + if err != nil { + return xerrors.Errorf("storage provider %s had invalid multiaddrs in their info: %w", maddr, err) + } + maddrs = append(maddrs, ma) + } + if len(maddrs) == 0 { + return xerrors.Errorf("storage provider %s has no multiaddrs set on-chain", maddr) + } + + addrInfo := &peer.AddrInfo{ + ID: *minfo.PeerId, + Addrs: maddrs, + } + + log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr) + + var hurls []*url.URL + + for _, ma := range addrInfo.Addrs { + hurl, err := maurl.ToURL(ma) + if err != nil { + return xerrors.Errorf("failed to convert multiaddr %s to URL: %w", ma, err) + } + if hurl.Scheme == "ws" { + hurl.Scheme = "http" + } + if hurl.Scheme == "wss" { + hurl.Scheme = "https" + } + log.Debugw("converted multiaddr to URL", "url", hurl, "multiaddr", ma.String()) + hurls = append(hurls, hurl) + } + + purl := hurls[0] + log.Debugw("using first URL", "url", purl) + tu := mk20.StartUpload{ + ChunkSize: chunkSize, + } + b, err := json.Marshal(tu) + if err != nil { + return err + } + log.Debugw("request body", "body", string(b)) + client, err := http.NewRequest("POST", purl.String()+"/market/mk20/upload/"+dealid.String(), bytes.NewBuffer(b)) + if err != nil { + return xerrors.Errorf("failed to upload start create request: %w", err) + } + client.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(client) + if err != nil { + return xerrors.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusTooManyRequests { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + + x, err := mmap.Open(f.Name()) + if err != nil { + return xerrors.Errorf("failed to open file: %w", err) + } + defer x.Close() + + for { + resp, err = http.Get(purl.String() + "/market/mk20/upload/" + dealid.String()) + if err != nil { + return xerrors.Errorf("failed to send request: %w", err) + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + + ustatus := mk20.UploadStatus{} + err = json.Unmarshal(respBody, &ustatus) + if err != nil { + return xerrors.Errorf("failed to unmarshal response body: %w", err) + } + + log.Debugw("upload status", "status", ustatus) + + if ustatus.TotalChunks != numChunks { + return xerrors.Errorf("expected %d chunks, got %d", numChunks, ustatus.TotalChunks) + } + + if ustatus.Missing == 0 { + break + } + + log.Warnw("missing chunks", "missing", ustatus.Missing) + // Try to upload missing chunks + for _, c := range ustatus.MissingChunks { + start := int64(c-1) * chunkSize + end := start + chunkSize + if end > size { + end = size + } + log.Debugw("uploading chunk", "start", start, "end", end) + buf := make([]byte, end-start) + _, err := x.ReadAt(buf, start) + if err != nil { + return xerrors.Errorf("failed to read chunk: %w", err) + } + req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/upload/"+dealid.String()+"/"+fmt.Sprintf("%d", c), bytes.NewBuffer(buf)) + if err != nil { + return xerrors.Errorf("failed to create put request: %w", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprintf("%d", end-start)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("failed to send put request: %w", err) + } + if resp.StatusCode != http.StatusOK { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + } + } + + log.Infow("upload complete") + + //Finalize the upload + resp, err = http.Post(purl.String()+"/market/mk20/upload/finalize/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) + if err != nil { + return xerrors.Errorf("failed to send request: %w", err) + } + if resp.StatusCode != http.StatusOK { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + + return nil + }, +} diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 3addd52c1..358473a37 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -1092,10 +1092,22 @@ When the cumulative size of all deals in process reaches this number, new deals If True then all deals coming from unknown clients will be rejected. (Default: false)`, }, { - Name: "MaxParallelUploads", + Name: "MaxParallelChunkUploads", Type: "int", - Comment: `MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16)`, + Comment: `MaxParallelChunkUploads defines the maximum number of upload operations that can run in parallel. (Default: 512)`, + }, + { + Name: "MinimumChunkSize", + Type: "int64", + + Comment: `MinimumChunkSize defines the smallest size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 16 MiB)`, + }, + { + Name: "MaximumChunkSize", + Type: "int64", + + Comment: `MaximumChunkSize defines the maximum size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 256 MiB)`, }, }, "MarketConfig": { diff --git a/deps/config/types.go b/deps/config/types.go index a4eb97894..ed991079a 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -113,7 +113,9 @@ func DefaultCurioConfig() *CurioConfig { MK20: MK20Config{ ExpectedPoRepSealDuration: 8 * time.Hour, ExpectedSnapSealDuration: 2 * time.Hour, - MaxParallelUploads: 16, + MaxParallelChunkUploads: 512, + MinimumChunkSize: 16 * 1024 * 1024, // 16 MiB + MaximumChunkSize: 256 * 1024 * 1024, // 256 MiB }, IPNI: IPNIConfig{ ServiceURL: []string{"https://cid.contact"}, @@ -910,6 +912,12 @@ type MK20Config struct { // If True then all deals coming from unknown clients will be rejected. (Default: false) DenyUnknownClients bool - // MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16) - MaxParallelUploads int + // MaxParallelChunkUploads defines the maximum number of upload operations that can run in parallel. (Default: 512) + MaxParallelChunkUploads int + + // MinimumChunkSize defines the smallest size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 16 MiB) + MinimumChunkSize int64 + + // MaximumChunkSize defines the maximum size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 256 MiB) + MaximumChunkSize int64 } diff --git a/docker/piece-server/sample/mk20-random-deal.sh b/docker/piece-server/sample/mk20-random-deal.sh index 1ff612f9c..6f8aa6028 100755 --- a/docker/piece-server/sample/mk20-random-deal.sh +++ b/docker/piece-server/sample/mk20-random-deal.sh @@ -4,9 +4,11 @@ set -e ci="\e[3m" cn="\e[0m" -offline="${1:=false}" -chunks="${2:-51200}" -links="${3:-100}" + +put="${1:-false}" +offline="${2:-false}" +chunks="${3:-51200}" +links="${4:-100}" printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` @@ -16,22 +18,36 @@ mv $FILE /var/lib/curio-client/data/$COMMP_CID miner_actor=$(lotus state list-miners | grep -v t01000) -if [ "$offline" == "true" ]; then - +if [ "$put" == "true" ]; then ################################################################################### - printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ - --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ - --contract-address 0xtest --contract-verify-method test\n\n${cn}" + printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ + --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --contract-address 0xtest --contract-verify-method test --put\n\n${cn}" - sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test --put else - ################################################################################### - printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ - --http-url=http://piece-server:12320/pieces?id=$COMMP_CID \ - --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ - --contract-address 0xtest --contract-verify-method test\n\n${cn}" - sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + if [ "$offline" == "true" ]; then + + ################################################################################### + printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ + --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --contract-address 0xtest --contract-verify-method test\n\n${cn}" + + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + + else + ################################################################################### + printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ + --http-url=http://piece-server:12320/pieces?id=$COMMP_CID \ + --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --contract-address 0xtest --contract-verify-method test\n\n${cn}" + + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + + fi + +fi + -fi \ No newline at end of file diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 28597493b..779497634 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -699,10 +699,20 @@ description: The default curio configuration # type: bool #DenyUnknownClients = false - # MaxParallelUploads defines the maximum number of upload operations that can run in parallel. (Default: 16) + # MaxParallelChunkUploads defines the maximum number of upload operations that can run in parallel. (Default: 512) # # type: int - #MaxParallelUploads = 16 + #MaxParallelChunkUploads = 512 + + # MinimumChunkSize defines the smallest size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 16 MiB) + # + # type: int64 + #MinimumChunkSize = 16777216 + + # MaximumChunkSize defines the maximum size of a chunk allowed for processing, expressed in bytes. Must be a power of 2. (Default: 256 MiB) + # + # type: int64 + #MaximumChunkSize = 268435456 # IPNI configuration for ipni-provider # diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 800e0f4c9..6a643eb06 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -902,6 +902,7 @@ COMMANDS: init Initialise curio mk12 client repo deal Make a mk20 deal with Curio aggregate Create a new aggregate from a list of CAR files + upload Upload a file to the storage provider help, h Shows a list of commands or help for one command OPTIONS: @@ -962,3 +963,18 @@ OPTIONS: --out output the aggregate file (default: false) --help, -h show help ``` + +#### sptool toolbox mk20-client upload +``` +NAME: + sptool toolbox mk20-client upload - Upload a file to the storage provider + +USAGE: + sptool toolbox mk20-client upload [command options] + +OPTIONS: + --provider value storage provider on-chain address + --deal value deal id to upload to + --chunk-size value chunk size to be used for the upload (default: "4 MiB") + --help, -h show help +``` diff --git a/go.mod b/go.mod index 82c849419..b28d5c415 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.14.0 github.com/multiformats/go-multibase v0.2.0 - github.com/multiformats/go-multicodec v0.9.0 + github.com/multiformats/go-multicodec v0.9.1 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-varint v0.0.7 github.com/oklog/ulid v1.3.1 diff --git a/go.sum b/go.sum index a00ac5eff..51a61bd6b 100644 --- a/go.sum +++ b/go.sum @@ -1075,8 +1075,8 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= -github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo= +github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index d10ab30ab..0aadbf86e 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -51,7 +51,7 @@ BEGIN indexed = CASE WHEN market_piece_metadata.indexed = FALSE THEN EXCLUDED.indexed ELSE market_piece_metadata.indexed - END; + END; -- Insert into the market_piece_deal table INSERT INTO market_piece_deal ( @@ -189,8 +189,7 @@ CREATE TABLE market_mk20_pipeline ( CREATE TABLE market_mk20_pipeline_waiting ( id TEXT PRIMARY KEY, - waiting_for_data BOOLEAN DEFAULT FALSE, - start_time TIMESTAMPTZ DEFAULT NULL + waiting_for_data BOOLEAN DEFAULT FALSE ); CREATE TABLE market_mk20_download_pipeline ( @@ -211,6 +210,17 @@ CREATE TABLE market_mk20_offline_urls ( PRIMARY KEY (id, piece_cid, piece_size) ); +CREATE TABLE market_mk20_deal_chunk ( + id TEXT not null, + chunk INT not null, + chunk_size BIGINT not null, + url TEXT DEFAULT NULL, + complete BOOLEAN DEFAULT FALSE, + finalize BOOLEAN DEFAULT FALSE, + finalize_task_id BIGINT DEFAULT NULL, + PRIMARY KEY (id, chunk) +); + CREATE TABLE market_mk20_products ( name TEXT PRIMARY KEY, enabled BOOLEAN DEFAULT TRUE diff --git a/lib/paths/http_handler.go b/lib/paths/http_handler.go index 61a603801..d04e46ce6 100644 --- a/lib/paths/http_handler.go +++ b/lib/paths/http_handler.go @@ -65,6 +65,7 @@ func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/remote/{type}/{id}/{spt}/allocated/{offset}/{size}", handler.remoteGetAllocated).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") + mux.HandleFunc("/remote/stash/{id}", handler.remoteGetSector).Methods("POST") mux.ServeHTTP(w, r) } diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 20f0ef08f..014c6b47c 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -36,8 +37,6 @@ var infoMarkdown []byte var log = logging.Logger("mk20httphdlr") -const maxPutBodySize int64 = 64 << 30 // 64 GiB - const requestTimeout = 10 * time.Second type MK20DealHandler struct { @@ -66,13 +65,16 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { func Router(mdh *MK20DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "timeout request")) - mux.Method("GET", "/status", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "timeout reading request")) - mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "timeout reading request")) - mux.Put("/data", mdh.mk20UploadDealData) - mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "timeout reading request")) - mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "timeout reading request")) - mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "timeout reading request")) + mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "request timeout")) + mux.Method("GET", "/status/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "request timeout")) + mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) + mux.Method("POST", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStart), requestTimeout, "request timeout")) + mux.Method("GET", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStatus), requestTimeout, "request timeout")) + mux.Put("/upload/{id}/{chunkNum}", mdh.mk20UploadDealChunks) + mux.Method("POST", "/upload/finalize/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20FinalizeUpload), requestTimeout, "request timeout")) + mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "request timeout")) + mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) + mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) return mux } @@ -123,8 +125,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { // mk20status handles HTTP requests to retrieve the status of a deal using its ID, responding with deal status or appropriate error codes. func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { - // Extract id from the URL - idStr := r.URL.Query().Get("id") + idStr := chi.URLParam(r, "id") if idStr == "" { log.Errorw("missing id in url", "url", r.URL) http.Error(w, "missing id in url", http.StatusBadRequest) @@ -187,40 +188,6 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt } } -// mk20UploadDealData handles uploading deal data to the server using a PUT request with specific validations and streams directly to the logic. -func (mdh *MK20DealHandler) mk20UploadDealData(w http.ResponseWriter, r *http.Request) { - // Extract id from the URL - idStr := r.URL.Query().Get("id") - if idStr == "" { - log.Errorw("missing id in url", "url", r.URL) - w.WriteHeader(http.StatusBadRequest) - return - } - - id, err := ulid.Parse(idStr) - if err != nil { - log.Errorw("invalid id in url", "id", idStr, "err", err) - w.WriteHeader(http.StatusBadRequest) - return - } - - // Check Content-Type - ct := r.Header.Get("Content-Type") - if ct == "" || !strings.HasPrefix(ct, "application/octet-stream") { - http.Error(w, "invalid or missing Content-Type", http.StatusUnsupportedMediaType) - return - } - - // validate Content-Length - if r.ContentLength <= 0 || r.ContentLength > maxPutBodySize { - http.Error(w, fmt.Sprintf("invalid Content-Length: %d", r.ContentLength), http.StatusRequestEntityTooLarge) - return - } - - // Stream directly to execution logic - mdh.dm.MK20Handler.HandlePutRequest(id, r.Body, w) -} - // info serves the contents of the info file as a text/markdown response with HTTP 200 or returns an HTTP 500 on read/write failure. func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { @@ -374,3 +341,117 @@ func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http. log.Errorw("failed to write supported sources", "err", err) } } + +func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Request) { + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + mdh.dm.MK20Handler.HandleUploadStatus(r.Context(), id, w) +} + +func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http.Request) { + ct := r.Header.Get("Content-Type") + if ct != "application/octet-stream" { + log.Errorw("invalid content type", "ct", ct) + http.Error(w, "invalid content type", http.StatusBadRequest) + return + } + + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + chunk := chi.URLParam(r, "chunkNum") + if chunk == "" { + log.Errorw("missing chunk number in url", "url", r.URL) + http.Error(w, "missing chunk number in url", http.StatusBadRequest) + return + } + + chunkNum, err := strconv.Atoi(chunk) + if err != nil { + log.Errorw("invalid chunk number in url", "url", r.URL) + http.Error(w, "invalid chunk number in url", http.StatusBadRequest) + return + } + + mdh.dm.MK20Handler.HandleUploadChunk(id, chunkNum, r.Body, w) +} + +func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Request) { + ct := r.Header.Get("Content-Type") + if ct != "application/json" { + log.Errorw("invalid content type", "ct", ct) + http.Error(w, "invalid content type", http.StatusBadRequest) + return + } + + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + reader := io.LimitReader(r.Body, 4*1024*1024) + b, err := io.ReadAll(reader) + if err != nil { + log.Errorw("failed to read request body", "err", err) + http.Error(w, "failed to read request body", http.StatusBadRequest) + return + } + + upload := mk20.StartUpload{} + err = json.Unmarshal(b, &upload) + if err != nil { + log.Errorw("failed to unmarshal request body", "err", err) + http.Error(w, "failed to unmarshal request body", http.StatusBadRequest) + return + } + + mdh.dm.MK20Handler.HandleUploadStart(r.Context(), id, upload.ChunkSize, w) + +} + +func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Request) { + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + mdh.dm.MK20Handler.HandleUploadFinalize(id, w) +} diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md index d121b28e9..ad8fd87ca 100644 --- a/market/mk20/http/info.md +++ b/market/mk20/http/info.md @@ -225,7 +225,7 @@ HttpUrl represents an HTTP endpoint configuration for fetching piece data. |-------|------|-----|-------------| | URL | [string](https://pkg.go.dev/builtin#string) | json:"url" | URL specifies the HTTP endpoint where the piece data can be fetched. | | Headers | [http.Header](https://pkg.go.dev/net/http#Header) | json:"headers" | HTTPHeaders represents the HTTP headers associated with the URL. | -| Priority | [uint64](https://pkg.go.dev/builtin#uint64) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | +| Priority | [int](https://pkg.go.dev/builtin#int) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | | Fallback | [bool](https://pkg.go.dev/builtin#bool) | json:"fallback" | Fallback indicates whether this URL serves as a fallback option when other URLs fail. | ### PieceDataFormat @@ -238,6 +238,14 @@ PieceDataFormat represents various formats in which piece data can be defined, i | Aggregate | [*mk20.FormatAggregate](#formataggregate) | json:"aggregate" | Aggregate holds a reference to the aggregated format of piece data. | | Raw | [*mk20.FormatBytes](#formatbytes) | json:"raw" | Raw represents the raw format of the piece data, encapsulated as bytes. | +### StartUpload + +StartUpload represents metadata for initiating an upload operation, containing the chunk size of the data to be uploaded. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| ChunkSize | [int64](https://pkg.go.dev/builtin#int64) | json:"chunk_size" | | + ### SupportedContracts SupportedContracts represents a collection of contract addresses supported by a system or application. @@ -267,6 +275,18 @@ SupportedProducts represents a collection of products supported by the SP. | Field | Type | Tag | Description | |-------|------|-----|-------------| +### UploadStatus + +UploadStatus represents the status of a file upload process, including progress and missing chunks. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| TotalChunks | [int](https://pkg.go.dev/builtin#int) | json:"total_chunks" | TotalChunks represents the total number of chunks required for the upload. | +| Uploaded | [int](https://pkg.go.dev/builtin#int) | json:"uploaded" | Uploaded represents the number of chunks successfully uploaded. | +| Missing | [int](https://pkg.go.dev/builtin#int) | json:"missing" | Missing represents the number of chunks that are not yet uploaded. | +| UploadedChunks | [[]int](https://pkg.go.dev/builtin#int) | json:"uploaded_chunks" | UploadedChunks is a slice containing the indices of successfully uploaded chunks. | +| MissingChunks | [[]int](https://pkg.go.dev/builtin#int) | json:"missing_chunks" | MissingChunks is a slice containing the indices of missing chunks. | + ### Constants for ErrorCode | Constant | Code | Description | diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index d78ebd05d..4239d4f60 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -44,12 +44,21 @@ type MK20 struct { sm map[address.Address]abi.SectorSize as *multictladdr.MultiAddressSelector stor paths.StashStore - maxParallelUploads *atomic.Int32 + maxParallelUploads *atomic.Int64 } func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, stor paths.StashStore) (*MK20, error) { ctx := context.Background() + // Ensure MinChunk size and max chunkSize is a power of 2 + if cfg.Market.StorageMarketConfig.MK20.MinimumChunkSize&(cfg.Market.StorageMarketConfig.MK20.MinimumChunkSize-1) != 0 { + return nil, xerrors.Errorf("MinimumChunkSize must be a power of 2") + } + + if cfg.Market.StorageMarketConfig.MK20.MaximumChunkSize&(cfg.Market.StorageMarketConfig.MK20.MaximumChunkSize-1) != 0 { + return nil, xerrors.Errorf("MaximumChunkSize must be a power of 2") + } + sm := make(map[address.Address]abi.SectorSize) for _, m := range miners { @@ -72,7 +81,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI sm: sm, as: as, stor: stor, - maxParallelUploads: new(atomic.Int32), + maxParallelUploads: new(atomic.Int64), }, nil } diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go new file mode 100644 index 000000000..2f1a9092d --- /dev/null +++ b/market/mk20/mk20_upload.go @@ -0,0 +1,362 @@ +package mk20 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "os" + "strings" + "time" + + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/dealdata" +) + +func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.ResponseWriter) { + var exists bool + err := m.db.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_pipeline_waiting + WHERE id = $1 AND waiting_for_data = TRUE + )`, id.String()).Scan(&exists) + if err != nil { + log.Errorw("failed to check if upload is waiting for data", "deal", id, "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + if !exists { + http.Error(w, "deal not found", http.StatusNotFound) + return + } + + var ret UploadStatus + + err = m.db.QueryRow(ctx, `SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE complete) AS complete, + COUNT(*) FILTER (WHERE NOT complete) AS missing, + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE complete) AS completed_chunks, + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE NOT complete) AS incomplete_chunks + FROM + market_mk20_deal_chunk + WHERE + id = $1 + GROUP BY + id;`, id.String()).Scan(&ret.TotalChunks, &ret.Uploaded, &ret.Missing, &ret.UploadedChunks, &ret.MissingChunks) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + log.Errorw("failed to get upload status", "deal", id, "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + http.Error(w, "chunk size not updated", http.StatusNotFound) + return + } + + data, err := json.Marshal(ret) + if err != nil { + log.Errorw("failed to marshal upload status", "deal", id, "error", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + _, err = w.Write(data) + if err != nil { + log.Errorw("failed to write upload status", "deal", id, "error", err) + } +} + +func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, chunkSize int64, w http.ResponseWriter) { + if chunkSize == 0 { + log.Errorw("chunk size must be greater than 0", "id", id) + http.Error(w, "chunk size must be greater than 0", http.StatusBadRequest) + return + } + + // Check if chunk size is a power of 2 + if chunkSize&(chunkSize-1) != 0 { + log.Errorw("chunk size must be a power of 2", "id", id) + http.Error(w, "chunk size must be a power of 2", http.StatusBadRequest) + return + } + + // Check that chunk size align with config + if chunkSize < m.cfg.Market.StorageMarketConfig.MK20.MinimumChunkSize { + log.Errorw("chunk size too small", "id", id) + http.Error(w, "chunk size too small", http.StatusBadRequest) + return + } + if chunkSize > m.cfg.Market.StorageMarketConfig.MK20.MaximumChunkSize { + log.Errorw("chunk size too large", "id", id) + http.Error(w, "chunk size too large", http.StatusBadRequest) + return + } + + // Check if deal exists + var exists bool + + err := m.db.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_deal + WHERE id = $1 + );`, id.String()).Scan(&exists) + if err != nil { + log.Errorw("failed to check if deal exists", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + if !exists { + http.Error(w, "deal not found", http.StatusNotFound) + return + } + + // Check if we already started the upload + var started bool + err = m.db.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_deal_chunk + WHERE id = $1);`, id.String()).Scan(&started) + if err != nil { + log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if started { + http.Error(w, "deal upload has already started", http.StatusTooManyRequests) + return + } + + deal, err := DealFromDB(ctx, m.db, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + rawSize, err := deal.Data.RawSize() + if err != nil { + log.Errorw("failed to get raw size of deal", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + numChunks := int(math.Ceil(float64(rawSize) / float64(chunkSize))) + + // Create rows in market_mk20_deal_chunk for each chunk for the ID + comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + batch := &pgx.Batch{} + batchSize := 15000 + for i := 1; i <= numChunks; i++ { + if i < numChunks { + batch.Queue(`INSERT INTO market_mk20_deal_chunk (id, chunk, chunk_size, complete) VALUES ($1, $2, $3, FALSE);`, id.String(), i, chunkSize) + } else { + // Calculate the size of last chunk + s := int64(rawSize) - (int64(numChunks-1) * chunkSize) + if s <= 0 || s > chunkSize { + return false, xerrors.Errorf("invalid chunk size") + } + + batch.Queue(`INSERT INTO market_mk20_deal_chunk (id, chunk, chunk_size, complete) VALUES ($1, $2, $3, FALSE);`, id.String(), i, s) + } + if batch.Len() >= batchSize { + res := tx.SendBatch(ctx, batch) + if err := res.Close(); err != nil { + return false, xerrors.Errorf("closing insert chunk batch: %w", err) + } + batch = &pgx.Batch{} + } + } + if batch.Len() > 0 { + res := tx.SendBatch(ctx, batch) + if err := res.Close(); err != nil { + return false, xerrors.Errorf("closing insert chunk batch: %w", err) + } + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorw("failed to create chunks for deal", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + if !comm { + log.Errorw("failed to create chunks for deal", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) +} + +func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w http.ResponseWriter) { + ctx := context.Background() + defer data.Close() + + if chunk < 1 { + http.Error(w, "chunk must be greater than 0", http.StatusBadRequest) + return + } + + var chunkDetails []struct { + Chunk int `db:"chunk"` + Size int64 `db:"chunk_size"` + Complete bool `db:"complete"` + } + err := m.db.Select(ctx, &chunkDetails, `SELECT chunk, chunk_size, complete + FROM market_mk20_deal_chunk + WHERE id = $1 AND chunk = $2`, id.String(), chunk) + if err != nil { + log.Errorw("failed to check if chunk exists", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + } + + if len(chunkDetails) == 0 { + http.Error(w, "chunk not found", http.StatusNotFound) + return + } + + if len(chunkDetails) > 1 { + log.Errorw("chunk exists multiple times", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if chunkDetails[0].Complete { + http.Error(w, "chunk already uploaded", http.StatusConflict) + return + } + + defer func() { + m.maxParallelUploads.Add(-1) + }() + + log.Debugw("uploading chunk", "deal", id, "chunk", chunk) + + chunkSize := chunkDetails[0].Size + reader := NewTimeoutReader(data, time.Second*5) + m.maxParallelUploads.Add(1) + + // Function to write data into StashStore and calculate commP + writeFunc := func(f *os.File) error { + limitedReader := io.LimitReader(reader, chunkSize+1) // +1 to detect exceeding the limit + + size, err := io.CopyBuffer(f, limitedReader, make([]byte, 4<<20)) + if err != nil { + return fmt.Errorf("failed to read and write chunk data: %w", err) + } + + if size > chunkSize { + return fmt.Errorf("chunk data exceeds the maximum allowed size") + } + + if chunkSize != size { + return fmt.Errorf("chunk size %d does not match with uploaded data size %d", chunkSize, size) + } + + return nil + } + + // Upload into StashStore + stashID, err := m.stor.StashCreate(ctx, chunkSize, writeFunc) + if err != nil { + if err.Error() == "chunk data exceeds the maximum allowed size" { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, "chunk data exceeds the maximum allowed size", http.StatusRequestEntityTooLarge) + return + } else if strings.Contains(err.Error(), "does not match with uploaded data") { + log.Errorw("Storing", "Deal", id, "error", err) + http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) + return + } else { + log.Errorw("Failed to store piece data in StashStore", "error", err) + http.Error(w, "Failed to store piece data", http.StatusInternalServerError) + return + } + } + + log.Debugw("uploaded chunk", "deal", id, "chunk", chunk, "stashID", stashID.String()) + + stashUrl, err := m.stor.StashURL(stashID) + if err != nil { + log.Errorw("Failed to get stash url", "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + stashUrl.Scheme = dealdata.CustoreScheme + + log.Debugw("uploading chunk generated URL", "deal", id, "chunk", chunk, "url", stashUrl.String()) + + n, err := m.db.Exec(ctx, `UPDATE market_mk20_deal_chunk SET complete = TRUE, + url = $1 + WHERE id = $2 + AND chunk = $3 + AND complete = FALSE`, stashUrl.String(), id.String(), chunk) + if err != nil { + log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + err = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to remove stash file", "Deal", id, "error", err) + } + return + } + + if n == 0 { + log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + err = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to remove stash file", "Deal", id, "error", err) + } + return + } + +} + +func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { + ctx := context.Background() + var exists bool + err := m.db.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_deal_chunk + WHERE id = $1 AND complete = FALSE OR complete IS NULL + )`, id.String()).Scan(&exists) + if err != nil { + log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if exists { + http.Error(w, "deal upload has not finished", http.StatusBadRequest) + return + } + + n, err := m.db.Exec(ctx, `UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) + if err != nil { + log.Errorw("failed to finalize deal upload", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if n == 0 { + log.Errorw("failed to finalize deal upload", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index de4e3a3c7..eb06c1a9a 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -7,21 +7,10 @@ import ( "fmt" "io" "net/http" - "os" - "strings" "time" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" - "golang.org/x/xerrors" - - "github.com/filecoin-project/go-address" - commcid "github.com/filecoin-project/go-fil-commcid" - commp "github.com/filecoin-project/go-fil-commp-hashhash" - "github.com/filecoin-project/go-state-types/abi" - - "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/lib/dealdata" ) func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { @@ -136,281 +125,6 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { } } -func (m *MK20) HandlePutRequest(id ulid.ULID, data io.ReadCloser, w http.ResponseWriter) { - ctx := context.Background() - defer data.Close() - - var ( - idStr string - updated bool - ) - - err := m.db.QueryRow(ctx, `WITH check_row AS ( - SELECT id FROM market_mk20_pipeline_waiting - WHERE id = $1 AND waiting_for_data = TRUE - ), - try_update AS ( - UPDATE market_mk20_pipeline_waiting - SET start_time = NOW() - WHERE id IN (SELECT id FROM check_row) - AND ( - start_time IS NULL - OR start_time < NOW() - ($2 * INTERVAL '1 second') - ) - RETURNING id - ) - SELECT - check_row.id, - try_update.id IS NOT NULL AS updated - FROM check_row - LEFT JOIN try_update ON check_row.id = try_update.id;`, - id.String(), m.cfg.HTTP.ReadTimeout.Seconds()).Scan(&idStr, &updated) - - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - http.Error(w, "", http.StatusNotFound) - return - } - log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - if idStr != id.String() { - log.Errorw("deal id mismatch", "deal", id.String(), "db", idStr) - http.Error(w, "", http.StatusInternalServerError) - return - } - - if !updated { - http.Error(w, "", http.StatusConflict) - } - - deal, err := DealFromDB(ctx, m.db, id) - if err != nil { - log.Errorw("failed to get deal from db", "deal", id.String(), "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - rawSize, err := deal.Data.RawSize() - if err != nil { - log.Errorw("failed to get raw size of deal", "deal", id.String(), "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - n, err := m.db.Exec(ctx, `UPDATE market_mk20_pipeline_waiting SET started_put = TRUE, start_time = NOW() WHERE id = $1`, id.String()) - if err != nil { - log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - if n != 1 { - log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - failed := true - - defer func() { - m.maxParallelUploads.Add(-1) - if failed { - _, err = m.db.Exec(ctx, `UPDATE market_mk20_pipeline_waiting SET started_put = FALSE, start_time = NULL WHERE id = $1`, id.String()) - if err != nil { - log.Errorw("failed to update deal status in db", "deal", id.String(), "err", err) - } - } - }() - - if m.maxParallelUploads.Load() >= int32(m.cfg.Market.StorageMarketConfig.MK20.MaxParallelUploads) { - http.Error(w, "Too many parallel uploads", http.StatusTooManyRequests) - return - } - - m.maxParallelUploads.Add(1) - - cp := new(commp.Calc) - reader := NewTimeoutReader(data, time.Second*5) - - // Function to write data into StashStore and calculate commP - writeFunc := func(f *os.File) error { - limitedReader := io.LimitReader(reader, int64(rawSize+1)) // +1 to detect exceeding the limit - wr := io.MultiWriter(f, cp) - - size, err := io.CopyBuffer(wr, limitedReader, make([]byte, 4<<20)) - if err != nil { - return fmt.Errorf("failed to read and write piece data: %w", err) - } - - if size > int64(deal.Data.Size) { - return fmt.Errorf("piece data exceeds the maximum allowed size") - } - - if int64(rawSize) != size { - return fmt.Errorf("deal raw size %d does not match with uploaded data size %d", rawSize, size) - } - - digest, pieceSize, err := cp.Digest() - if err != nil { - return fmt.Errorf("failed to calculate commP: %w", err) - } - - pieceCIDComputed, err := commcid.DataCommitmentV1ToCID(digest) - if err != nil { - return fmt.Errorf("failed to calculate piece CID: %w", err) - } - if !pieceCIDComputed.Equals(deal.Data.PieceCID) { - return fmt.Errorf("calculated piece CID %s does not match with deal piece CID %s", pieceCIDComputed.String(), deal.Data.PieceCID.String()) - } - - if abi.PaddedPieceSize(pieceSize) != deal.Data.Size { - return fmt.Errorf("calculated piece size %d does not match with deal piece size %d", pieceSize, deal.Data.Size) - } - - return nil - } - - // Upload into StashStore - stashID, err := m.stor.StashCreate(ctx, int64(deal.Data.Size), writeFunc) - if err != nil { - if err.Error() == "piece data exceeds the maximum allowed size" { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, "piece data exceeds the maximum allowed size", http.StatusRequestEntityTooLarge) - return - } else if strings.Contains(err.Error(), "does not match with uploaded data") { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) - return - } else if strings.Contains(err.Error(), "failed to calculate piece CID") { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, "Failed to calculate piece CID", http.StatusInternalServerError) - return - } else if strings.Contains(err.Error(), "calculated piece CID does not match with uploaded data") { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) - return - } else if strings.Contains(err.Error(), "calculated piece size does not match with uploaded data") { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) - return - } else { - log.Errorw("Failed to store piece data in StashStore", "error", err) - http.Error(w, "Failed to store piece data", http.StatusInternalServerError) - return - } - } - - comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - - // 1. Create a long-term parked piece entry - var parkedPieceID int64 - err := tx.QueryRow(` - INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - VALUES ($1, $2, $3, TRUE) RETURNING id - `, deal.Data.PieceCID.String(), deal.Data.Size, rawSize).Scan(&parkedPieceID) - if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) - } - - // 2. Create a piece ref with data_url being "stashstore://" - // Get StashURL - stashURL, err := m.stor.StashURL(stashID) - if err != nil { - return false, fmt.Errorf("failed to get stash URL: %w", err) - } - - stashURL.Scheme = dealdata.CustoreScheme - dataURL := stashURL.String() - - var pieceRefID int64 - err = tx.QueryRow(` - INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, TRUE) RETURNING ref_id - `, parkedPieceID, dataURL).Scan(&pieceRefID) - if err != nil { - return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) - } - - n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - id.String(), deal.Data.PieceCID.String(), deal.Data.Size, []int64{pieceRefID}) - if err != nil { - return false, xerrors.Errorf("inserting mk20 download pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting mk20 download pipeline: %d rows affected", n) - } - - spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) - if err != nil { - return false, fmt.Errorf("getting provider ID: %w", err) - } - - ddo := deal.Products.DDOV1 - dealdata := deal.Data - dealID := deal.Identifier.String() - - var allocationID interface{} - if ddo.AllocationId != nil { - allocationID = *ddo.AllocationId - } else { - allocationID = nil - } - - aggregation := 0 - if dealdata.Format.Aggregate != nil { - aggregation = int(dealdata.Format.Aggregate.Type) - } - - n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, - piece_size, raw_size, offline, indexing, announce, - allocation_id, duration, piece_aggregation, started, after_commp) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE, TRUE)`, - dealID, spid, ddo.ContractAddress, ddo.Client.String(), dealdata.PieceCID.String(), - dealdata.Size, int64(dealdata.SourceHttpPut.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, - allocationID, ddo.Duration, aggregation) - if err != nil { - return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) - } - - _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) - if err != nil { - return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) - } - - return true, nil // Commit the transaction - }, harmonydb.OptionRetry()) - - if err != nil { - log.Errorw("Failed to process piece upload", "Deal", id, "error", err) - http.Error(w, "Failed to process piece upload", http.StatusInternalServerError) - err = m.stor.StashRemove(ctx, stashID) - if err != nil { - log.Errorw("Failed to remove stash file", "Deal", id, "error", err) - } - return - } - - if !comm { - http.Error(w, "Failed to process piece upload", http.StatusInternalServerError) - err = m.stor.StashRemove(ctx, stashID) - if err != nil { - log.Errorw("Failed to remove stash file", "Deal", id, "error", err) - } - return - } - - failed = false - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("OK")) -} - func (m *MK20) Supported(ctx context.Context) (map[string]bool, map[string]bool, error) { var products []struct { Name string `db:"name"` @@ -465,9 +179,6 @@ func (t *TimeoutReader) Read(p []byte) (int, error) { } if n > 0 { - // Successfully read some data; reset the deadline - deadline = time.Now().Add(t.timeout) - // Otherwise return bytes read and no error return n, err } diff --git a/market/mk20/types.go b/market/mk20/types.go index 5d3bb6707..0e7f8a03b 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -115,7 +115,7 @@ type HttpUrl struct { Headers http.Header `json:"headers"` // Priority indicates the order preference for using the URL in requests, with lower values having higher priority. - Priority uint64 `json:"priority"` + Priority int `json:"priority"` // Fallback indicates whether this URL serves as a fallback option when other URLs fail. Fallback bool `json:"fallback"` @@ -128,7 +128,7 @@ type DataSourceHttpPut struct { } // AggregateType represents an unsigned integer used to define the type of aggregation for data pieces in the system. -type AggregateType uint64 +type AggregateType int const ( diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 28e7049bd..31581344b 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -647,3 +647,27 @@ type SupportedDataSources struct { // Contracts represents a list of supported contract addresses in string format. Sources []string `json:"sources"` } + +// StartUpload represents metadata for initiating an upload operation, containing the chunk size of the data to be uploaded. +type StartUpload struct { + ChunkSize int64 `json:"chunk_size"` +} + +// UploadStatus represents the status of a file upload process, including progress and missing chunks. +type UploadStatus struct { + + // TotalChunks represents the total number of chunks required for the upload. + TotalChunks int `json:"total_chunks"` + + // Uploaded represents the number of chunks successfully uploaded. + Uploaded int `json:"uploaded"` + + // Missing represents the number of chunks that are not yet uploaded. + Missing int `json:"missing"` + + // UploadedChunks is a slice containing the indices of successfully uploaded chunks. + UploadedChunks []int `json:"uploaded_chunks"` + + //MissingChunks is a slice containing the indices of missing chunks. + MissingChunks []int `json:"missing_chunks"` +} diff --git a/pdp/handlers_upload.go b/pdp/handlers_upload.go index c53c5cbe7..e9eee9529 100644 --- a/pdp/handlers_upload.go +++ b/pdp/handlers_upload.go @@ -6,6 +6,7 @@ import ( "database/sql" "encoding/hex" "encoding/json" + "errors" "fmt" "hash" "io" @@ -84,7 +85,7 @@ func (ph *PieceHash) commp(ctx context.Context, db *harmonydb.DB) (cid.Cid, bool SELECT commp FROM pdp_piece_mh_to_commp WHERE mhash = $1 AND size = $2 `, mh, ph.Size).Scan(&commpStr) if err != nil { - if err == pgx.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { return cid.Undef, false, nil } return cid.Undef, false, fmt.Errorf("failed to query pdp_piece_mh_to_commp: %w", err) diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go new file mode 100644 index 000000000..73063adf0 --- /dev/null +++ b/tasks/piece/task_aggregate_chunks.go @@ -0,0 +1,341 @@ +package piece + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-commp-utils/writer" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/dealdata" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/market/mk20" +) + +type AggregateChunksTask struct { + db *harmonydb.DB + stor paths.StashStore + remote *paths.Remote +} + +func NewAggregateChunksTask(db *harmonydb.DB, stor paths.StashStore, remote *paths.Remote) *AggregateChunksTask { + return &AggregateChunksTask{ + db: db, + stor: stor, + remote: remote, + } +} + +func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var chunks []struct { + ID string `db:"id"` + Chunk int `db:"chunk"` + Size int64 `db:"chunk_size"` + URL string `db:"url"` + } + + err = a.db.Select(ctx, &chunks, ` + SELECT + id, + chunk, + chunk_size, + url + FROM + market_mk20_deal_chunk + WHERE + finalize_task_id = $1 + AND complete = TRUE + AND finalize = TRUE + ORDER BY chunk ASC`, taskID) + if err != nil { + return false, xerrors.Errorf("getting chunk details: %w", err) + } + + if len(chunks) == 0 { + return false, xerrors.Errorf("no chunks to aggregate for task %d", taskID) + } + + idStr := chunks[0].ID + + var isMk20 bool + var id ulid.ULID + var uid uuid.UUID + uid, err = uuid.Parse(idStr) + if err != nil { + serr := err + id, err = ulid.Parse(idStr) + if err != nil { + return false, xerrors.Errorf("parsing deal ID: %w, %w", serr, err) + } + isMk20 = true + } + + var rawSize int64 + var pcid cid.Cid + var psize abi.PaddedPieceSize + var deal *mk20.Deal + + if isMk20 { + deal, err = mk20.DealFromDB(ctx, a.db, id) + if err != nil { + return false, xerrors.Errorf("getting deal details: %w", err) + } + raw, err := deal.Data.RawSize() + if err != nil { + return false, xerrors.Errorf("getting deal raw size: %w", err) + } + rawSize = int64(raw) + pcid = deal.Data.PieceCID + psize = deal.Data.Size + } else { + rawSize = 4817498192 // TODO: Fix this for PDP + fmt.Println(uid) + } + + var readers []io.Reader + + for _, chunk := range chunks { + goUrl, err := url.Parse(chunk.URL) + if err != nil { + return false, xerrors.Errorf("parsing data URL: %w", err) + } + + upr := dealdata.NewUrlReader(a.remote, goUrl.String(), nil, chunk.Size) + + reader := upr + + defer func() { + _ = upr.Close() + }() + readers = append(readers, reader) + } + + rd := io.MultiReader(readers...) + + w := &writer.Writer{} + + // Function to write data into StashStore and calculate commP + writeFunc := func(f *os.File) error { + limitReader := io.LimitReader(rd, rawSize) + + multiWriter := io.MultiWriter(w, f) + + n, err := io.CopyBuffer(multiWriter, limitReader, make([]byte, writer.CommPBuf)) + if err != nil { + return fmt.Errorf("failed to read and write aggregated piece data: %w", err) + } + + if n != rawSize { + return fmt.Errorf("number of bytes written to CommP writer %d not equal to the file size %d", n, rawSize) + } + + return nil + } + + stashID, err := a.stor.StashCreate(ctx, rawSize, writeFunc) + if err != nil { + return false, xerrors.Errorf("stashing aggregated piece data: %w", err) + } + + calculatedCommp, err := w.Sum() + if err != nil { + return false, xerrors.Errorf("computing commP failed: %w", err) + } + + if !calculatedCommp.PieceCID.Equals(pcid) { + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", calculatedCommp.PieceCID.String(), pcid.String()) + } + + if calculatedCommp.PieceSize != psize { + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", calculatedCommp.PieceSize, psize) + } + + stashUrl, err := a.stor.StashURL(stashID) + if err != nil { + return false, xerrors.Errorf("getting stash URL: %w", err) + } + stashUrl.Scheme = dealdata.CustoreScheme + + comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var parkedPieceID int64 + + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, TRUE) RETURNING id + `, calculatedCommp.PieceCID.String(), calculatedCommp.PieceSize, rawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } + + var pieceRefID int64 + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, stashUrl.String()).Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + if isMk20 { + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, + id.String(), deal.Data.PieceCID.String(), deal.Data.Size, []int64{pieceRefID}) + if err != nil { + return false, xerrors.Errorf("inserting mk20 download pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 download pipeline: %d rows affected", n) + } + + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return false, fmt.Errorf("getting provider ID: %w", err) + } + + ddo := deal.Products.DDOV1 + dealdata := deal.Data + dealID := deal.Identifier.String() + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + aggregation := 0 + if dealdata.Format.Aggregate != nil { + aggregation = int(dealdata.Format.Aggregate.Type) + } + + n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid, + piece_size, raw_size, offline, indexing, announce, + allocation_id, duration, piece_aggregation, started, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE, TRUE)`, + dealID, spid, ddo.ContractAddress, ddo.Client.String(), dealdata.PieceCID.String(), + dealdata.Size, int64(dealdata.SourceHttpPut.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, + allocationID, ddo.Duration, aggregation) + if err != nil { + return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) + } + } else { + return false, xerrors.Errorf("not implemented for PDP") + // TODO: Do what is required for PDP + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("saving aggregated chunk details to DB: %w", err) + } + + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") + } + return true, nil +} + +func (a *AggregateChunksTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (a *AggregateChunksTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "AggregateChunks", + Cost: resources.Resources{ + Cpu: 1, + Ram: 4 << 30, + }, + MaxFailures: 3, + IAmBored: passcall.Every(30*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return a.schedule(context.Background(), taskFunc) + }), + } +} + +func (a *AggregateChunksTask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + // schedule submits + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + var mid string + var count int + err := a.db.QueryRow(ctx, `SELECT id, COUNT(*) AS total_chunks + FROM market_mk20_deal_chunk + GROUP BY id + HAVING + COUNT(*) = COUNT(*) FILTER ( + WHERE complete = TRUE + AND finalize = TRUE + AND finalize_task_id IS NULL + AND url IS NOT NULL + ) + ORDER BY id + LIMIT 1;`).Scan(&mid, &count) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("getting next task to schedule: %w", err) + } + if mid == "" { + return false, xerrors.Errorf("no id for tasks to schedule") + } + + n, err := tx.Exec(`UPDATE market_mk20_deal_chunk SET finalize_task_id = $1 + WHERE id = $2 + AND complete = TRUE + AND finalize = TRUE + AND finalize_task_id IS NULL + AND url IS NOT NULL`, id, mid) + if err != nil { + return false, xerrors.Errorf("updating chunk finalize task: %w", err) + } + if n != count { + return false, xerrors.Errorf("expected to update %d rows: %d rows affected", count, n) + } + stop = false + return true, nil + }) + } + return nil +} + +func (a *AggregateChunksTask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ = harmonytask.Reg(&AggregateChunksTask{}) +var _ harmonytask.TaskInterface = &AggregateChunksTask{} diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index a805493c6..2ea4ebafa 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -585,7 +585,7 @@ func (a *WebRPC) AddMarketContract(ctx context.Context, contract, abiString stri return fmt.Errorf("invalid abi: %w", err) } - if ethabi.Methods == nil || len(ethabi.Methods) == 0 { + if len(ethabi.Methods) == 0 { return fmt.Errorf("invalid abi: no methods") } @@ -621,7 +621,7 @@ func (a *WebRPC) UpdateMarketContract(ctx context.Context, contract, abiString s return fmt.Errorf("invalid abi: %w", err) } - if ethabi.Methods == nil || len(ethabi.Methods) == 0 { + if len(ethabi.Methods) == 0 { return fmt.Errorf("invalid abi: no methods") } From 6fa55ebf0a8bbeeef78fdc6493b812352b108560 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 11 Jun 2025 20:23:26 +0400 Subject: [PATCH 14/55] fix actor info page --- web/api/webrpc/actor_charts.go | 26 +++++++++++++++++-------- web/static/pages/actor/actor-detail.mjs | 21 +++++++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/web/api/webrpc/actor_charts.go b/web/api/webrpc/actor_charts.go index 4e136b870..42adeae76 100644 --- a/web/api/webrpc/actor_charts.go +++ b/web/api/webrpc/actor_charts.go @@ -2,6 +2,7 @@ package webrpc import ( "context" + "fmt" "sort" "github.com/samber/lo" @@ -29,14 +30,16 @@ type SectorBucket struct { } type SectorBuckets struct { - All []SectorBucket - CC []SectorBucket + All []SectorBucket + CC []SectorBucket + BlockDelaySeconds int } func (a *WebRPC) ActorCharts(ctx context.Context, maddr address.Address) (*SectorBuckets, error) { out := SectorBuckets{ - All: []SectorBucket{}, - CC: []SectorBucket{}, + All: []SectorBucket{}, + CC: []SectorBucket{}, + BlockDelaySeconds: int(build.BlockDelaySecs), } stor := store.ActorStore(ctx, @@ -106,8 +109,13 @@ func (a *WebRPC) ActorCharts(ctx context.Context, maddr address.Address) (*Secto if sector.VerifiedDealWeight.GreaterThan(abi.NewStoragePower(0)) { weight = big.Div(big.Mul(sector.VerifiedDealWeight, big.NewInt(verifiedPowerGainMul)), big.NewInt(int64(sector.Expiration-sector.PowerBaseEpoch))) } + + fmt.Println("Sector Number", sector.SectorNumber, "Weight", weight) + sb.QAP = big.Add(sb.QAP, weight) + fmt.Println("Sector Number", sector.SectorNumber, "QAP", sb.QAP) + if sector.DealWeight.Equals(abi.NewStoragePower(0)) && sector.VerifiedDealWeight.Equals(abi.NewStoragePower(0)) { sbc, ok := bucketsMapCC[bucket] if !ok { @@ -162,7 +170,7 @@ func (a *WebRPC) prepExpirationBucket(out []SectorBucket, now *types.TipSet) ([] totalCount := lo.Reduce(out, func(acc int64, b SectorBucket, _ int) int64 { return acc + b.Count }, int64(0)) - totalPower := lo.Reduce(out, func(agg big.Int, b SectorBucket, _ int) big.Int { return big.Add(agg, b.QAP) }, big.Zero()) + //totalPower := lo.Reduce(out, func(agg big.Int, b SectorBucket, _ int) big.Int { return big.Add(agg, b.QAP) }, big.Zero()) if len(out) == 0 { return out, nil @@ -179,13 +187,15 @@ func (a *WebRPC) prepExpirationBucket(out []SectorBucket, now *types.TipSet) ([] } for i := range out { + fmt.Println("Bucket", i, "Epoch", out[i].BucketEpoch, "Count", out[i].Count, "QAP", out[i].QAP, "VestedLockedFunds", out[i].VestedLockedFunds) newTotal := totalCount - out[i].Count out[i].Count = newTotal totalCount = newTotal - newTotalPower := big.Sub(totalPower, out[i].QAP) - out[i].QAP = newTotalPower - totalPower = newTotalPower + //newTotalPower := big.Sub(totalPower, out[i].QAP) + //fmt.Println("Bucket", i, "New Total Power", newTotalPower.String()) + //out[i].QAP = newTotalPower + //totalPower = newTotalPower epochsToExpiry := out[i].BucketEpoch - now.Height() secsToExpiry := int64(epochsToExpiry) * int64(build.BlockDelaySecs) diff --git a/web/static/pages/actor/actor-detail.mjs b/web/static/pages/actor/actor-detail.mjs index 15f1e2f69..3c4247294 100644 --- a/web/static/pages/actor/actor-detail.mjs +++ b/web/static/pages/actor/actor-detail.mjs @@ -100,7 +100,7 @@ customElements.define('actor-detail', class Actor extends LitElement { Source Config Layers: - ${entry.CLayers.map(layer => html`${layer} `)} + ${actorInfo.Summary.CLayers.map(layer => html`${layer} `)} @@ -390,6 +390,7 @@ class ActorCharts extends LitElement { const firstAll = this.data.All[0]?.BucketEpoch ?? Infinity; const firstCC = this.data.CC[0]?.BucketEpoch ?? Infinity; const nowEpoch = Math.min(firstAll, firstCC); + const blockDelaySeconds = this.data.BlockDelaySeconds // --------------------------- // 1) EXPIRATION CHART (All vs. CC) @@ -425,7 +426,7 @@ class ActorCharts extends LitElement { }, ], }, - options: this.createChartOptions('Expiration (Count)', 'Count', nowEpoch, allExpData, ccExpData), + options: this.createChartOptions('Expiration (Count)', 'Count', nowEpoch, blockDelaySeconds, allExpData, ccExpData), }; if (!this.chartExpiration) { @@ -457,14 +458,14 @@ class ActorCharts extends LitElement { borderColor: 'rgb(255, 205, 86)', backgroundColor: 'rgba(255, 205, 86, 0.2)', borderWidth: 1, - stepped: true, + stepped: 'after', fill: true, pointRadius: 2, data: allQAPData, }, ], }, - options: this.createChartOptions('Quality-Adjusted Power', 'QAP', nowEpoch, allQAPData), + options: this.createChartOptions('Quality-Adjusted Power', 'QAP', nowEpoch, blockDelaySeconds, allQAPData), }; if (!this.chartQAP) { @@ -496,7 +497,7 @@ class ActorCharts extends LitElement { borderColor: 'rgb(153, 102, 255)', backgroundColor: 'rgba(153, 102, 255, 0.2)', borderWidth: 1, - stepped: true, + stepped: 'after', fill: true, pointRadius: 2, data: allLockedData, @@ -507,6 +508,7 @@ class ActorCharts extends LitElement { 'Vesting Locked Funds', 'Locked Funds (FIL)', nowEpoch, + blockDelaySeconds, allLockedData ), }; @@ -527,10 +529,11 @@ class ActorCharts extends LitElement { * @param {string} chartTitle - The chart title * @param {string} yTitle - Label for Y axis * @param {number} nowEpoch - The earliest epoch we consider "current" + * @param {number} blockDelaySeconds - The BlockDelaySeconds for the build * @param {Array} allData - The data array for the "All" set * @param {Array} [ccData] - Optional data array for the "CC" set */ - createChartOptions(chartTitle, yTitle, nowEpoch, allData, ccData = []) { + createChartOptions(chartTitle, yTitle, nowEpoch, blockDelaySeconds, allData, ccData = []) { return { responsive: true, maintainAspectRatio: false, @@ -546,12 +549,12 @@ class ActorCharts extends LitElement { callbacks: { label: (context) => { const epochVal = context.parsed.x; - const daysOffset = Math.round(((epochVal - nowEpoch) * 30) / 86400); + const daysOffset = Math.round(((epochVal - nowEpoch) * blockDelaySeconds) / 86400); const months = (daysOffset / 30).toFixed(1); let value; if (yTitle === 'QAP') { - value = this.toHumanBytes(context.parsed.y); // For QAP + value = toHumanBytes(context.parsed.y); // For QAP } else if (yTitle === 'Locked Funds (FIL)') { value = this.toHumanFIL(context.parsed.y); // For Vesting } else { @@ -576,7 +579,7 @@ class ActorCharts extends LitElement { }, ticks: { callback: (value) => { - const days = Math.round(((value - nowEpoch) * 30) / 86400); + const days = Math.round(((value - nowEpoch) * blockDelaySeconds) / 86400); return days + 'd'; }, font: { From 701a0bdb1987c2ea5ee3b57e092e847ed5a55bb6 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 11 Jun 2025 21:23:18 +0400 Subject: [PATCH 15/55] add some PDP metrics --- market/retrieval/piecehandler.go | 24 ++++++++++++-- market/retrieval/remoteblockstore/metric.go | 35 +++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/market/retrieval/piecehandler.go b/market/retrieval/piecehandler.go index 931b0eb5b..1d1db2022 100644 --- a/market/retrieval/piecehandler.go +++ b/market/retrieval/piecehandler.go @@ -44,6 +44,20 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { return } + // For PDP metrics check if this is a PDP piece + var isPDP bool + err = rp.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_piecerefs WHERE piece_cid = $1 LIMIT 1)`, pieceCid.String()).Scan(&isPDP) + if err != nil { + log.Errorf("failed to query the db for piece CID %s: %s", pieceCid, err) + w.WriteHeader(http.StatusInternalServerError) + stats.Record(ctx, remoteblockstore.HttpPieceByCid500ResponseCount.M(1)) + return + } + + if isPDP { + stats.Record(ctx, remoteblockstore.PDPPieceByCidRequestCount.M(1)) + } + // Get a reader over the piece reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pieceCid) if err != nil { @@ -74,8 +88,14 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { setHeaders(w, pieceCid, contentType, int64(size)) serveContent(w, r, reader) - stats.Record(ctx, remoteblockstore.HttpPieceByCid200ResponseCount.M(1)) - stats.Record(ctx, remoteblockstore.HttpPieceByCidRequestDuration.M(float64(time.Since(startTime).Milliseconds()))) + if isPDP { + stats.Record(ctx, remoteblockstore.PDPPieceByCid200ResponseCount.M(1)) + stats.Record(ctx, remoteblockstore.PDPPieceByCidRequestDuration.M(float64(time.Since(startTime).Milliseconds()))) + stats.Record(ctx, remoteblockstore.PDPPieceBytesServedCount.M(int64(size))) + } else { + stats.Record(ctx, remoteblockstore.HttpPieceByCid200ResponseCount.M(1)) + stats.Record(ctx, remoteblockstore.HttpPieceByCidRequestDuration.M(float64(time.Since(startTime).Milliseconds()))) + } } func setHeaders(w http.ResponseWriter, pieceCid cid.Cid, contentType string, size int64) { diff --git a/market/retrieval/remoteblockstore/metric.go b/market/retrieval/remoteblockstore/metric.go index 8c3ddba1a..865580cd0 100644 --- a/market/retrieval/remoteblockstore/metric.go +++ b/market/retrieval/remoteblockstore/metric.go @@ -28,13 +28,20 @@ var defaultMillisecondsDistribution = view.Distribution( var ( RetrievalInfo = stats.Int64("retrieval_info", "Arbitrary counter to tag node info to", stats.UnitDimensionless) - // piece + // piece (including PDP and sub pieces) HttpPieceByCidRequestCount = stats.Int64("http/piece_by_cid_request_count", "Counter of /piece/ requests", stats.UnitDimensionless) HttpPieceByCidRequestDuration = stats.Float64("http/piece_by_cid_request_duration_ms", "Time spent retrieving a piece by cid", stats.UnitMilliseconds) HttpPieceByCid200ResponseCount = stats.Int64("http/piece_by_cid_200_response_count", "Counter of /piece/ 200 responses", stats.UnitDimensionless) HttpPieceByCid400ResponseCount = stats.Int64("http/piece_by_cid_400_response_count", "Counter of /piece/ 400 responses", stats.UnitDimensionless) HttpPieceByCid404ResponseCount = stats.Int64("http/piece_by_cid_404_response_count", "Counter of /piece/ 404 responses", stats.UnitDimensionless) HttpPieceByCid500ResponseCount = stats.Int64("http/piece_by_cid_500_response_count", "Counter of /piece/ 500 responses", stats.UnitDimensionless) + + // pdp + PDPPieceByCidRequestCount = stats.Int64("pdp/piece_by_cid_request_count", "Counter of /piece/ requests for PDP", stats.UnitDimensionless) + PDPPieceByCidRequestDuration = stats.Float64("pdp/piece_by_cid_request_duration_ms", "Time spent retrieving a piece by cid for PDP", stats.UnitMilliseconds) + PDPPieceByCid200ResponseCount = stats.Int64("pdp/piece_by_cid_200_response_count", "Counter of /piece/ 200 responses for PDP", stats.UnitDimensionless) + PDPPieceBytesServedCount = stats.Int64("pdp/piece_bytes_served_count", "Counter of the number of bytes served by PDP since startup", stats.UnitBytes) + // Gateway HttpRblsGetRequestCount = stats.Int64("http/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) HttpRblsGetSuccessResponseCount = stats.Int64("http/rbls_get_success_response_count", "Counter of successful RemoteBlockstore Get responses", stats.UnitDimensionless) @@ -74,6 +81,26 @@ var ( Aggregation: view.Count(), } + PDPPieceByCidRequestCountView = &view.View{ + Measure: PDPPieceByCidRequestCount, + Aggregation: view.Count(), + } + + PDPPieceByCidRequestDurationView = &view.View{ + Measure: PDPPieceByCidRequestDuration, + Aggregation: defaultMillisecondsDistribution, + } + + PDPPieceByCid200ResponseCountView = &view.View{ + Measure: PDPPieceByCid200ResponseCount, + Aggregation: view.Count(), + } + + PDPPieceBytesServedCountView = &view.View{ + Measure: PDPPieceBytesServedCount, + Aggregation: view.Count(), + } + HttpRblsGetRequestCountView = &view.View{ Measure: HttpRblsGetRequestCount, Aggregation: view.Count(), @@ -116,7 +143,7 @@ var ( } ) -// CacheViews groups all cache-related default views. +// RetrievalViews groups all retrieval-related default views. func init() { err := view.Register( HttpPieceByCidRequestCountView, @@ -125,6 +152,10 @@ func init() { HttpPieceByCid400ResponseCountView, HttpPieceByCid404ResponseCountView, HttpPieceByCid500ResponseCountView, + PDPPieceByCidRequestCountView, + PDPPieceByCidRequestDurationView, + PDPPieceByCid200ResponseCountView, + PDPPieceBytesServedCountView, HttpRblsGetRequestCountView, HttpRblsGetSuccessResponseCountView, HttpRblsGetFailResponseCountView, From 1a2d0aa43feecd9bc17b275294b3b0f253f75087 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 16 Jun 2025 16:43:44 +0400 Subject: [PATCH 16/55] use piecepark for upload --- cmd/curio/tasks/tasks.go | 4 +- .../harmonydb/sql/20250505-market_mk20.sql | 3 +- lib/paths/db_index.go | 2 +- lib/paths/gomock_reflect_847096142/prog.go | 64 ------- lib/paths/index.go | 2 +- lib/paths/local.go | 19 +- market/mk20/mk20.go | 18 +- market/mk20/mk20_upload.go | 172 +++++++++++++++--- tasks/piece/task_aggregate_chunks.go | 45 +++-- tasks/piece/task_park_piece.go | 4 +- web/api/webrpc/market_20.go | 38 ++++ web/static/pages/mk20-deal/deal.mjs | 6 +- web/static/pages/upload-status/index.html | 31 ++++ web/static/pages/upload-status/status.mjs | 94 ++++++++++ 14 files changed, 392 insertions(+), 110 deletions(-) delete mode 100644 lib/paths/gomock_reflect_847096142/prog.go create mode 100644 web/static/pages/upload-status/index.html create mode 100644 web/static/pages/upload-status/status.mjs diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index ddcb03ef9..ead515eae 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -220,12 +220,12 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan { // Piece handling if cfg.Subsystems.EnableParkPiece { - parkPieceTask, err := piece2.NewParkPieceTask(db, must.One(slrLazy.Val()), cfg.Subsystems.ParkPieceMaxTasks) + parkPieceTask, err := piece2.NewParkPieceTask(db, must.One(slrLazy.Val()), stor, cfg.Subsystems.ParkPieceMaxTasks) if err != nil { return nil, err } cleanupPieceTask := piece2.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0) - aggregateChunksTask := piece2.NewAggregateChunksTask(db, lstor, stor) + aggregateChunksTask := piece2.NewAggregateChunksTask(db, lstor, stor, must.One(slrLazy.Val())) activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask, aggregateChunksTask) } } diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 0aadbf86e..a425876bc 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -214,7 +214,7 @@ CREATE TABLE market_mk20_deal_chunk ( id TEXT not null, chunk INT not null, chunk_size BIGINT not null, - url TEXT DEFAULT NULL, + ref_id BIGINT DEFAULT NULL, complete BOOLEAN DEFAULT FALSE, finalize BOOLEAN DEFAULT FALSE, finalize_task_id BIGINT DEFAULT NULL, @@ -310,3 +310,4 @@ $$ LANGUAGE plpgsql; + diff --git a/lib/paths/db_index.go b/lib/paths/db_index.go index e30ffbb76..b5cc10a46 100644 --- a/lib/paths/db_index.go +++ b/lib/paths/db_index.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/paths/alertinginterface" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/storage/sealer/fsutil" diff --git a/lib/paths/gomock_reflect_847096142/prog.go b/lib/paths/gomock_reflect_847096142/prog.go deleted file mode 100644 index 3ab1986e7..000000000 --- a/lib/paths/gomock_reflect_847096142/prog.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "encoding/gob" - "flag" - "fmt" - "os" - "path" - "reflect" - - "github.com/golang/mock/mockgen/model" - - pkg_ "github.com/filecoin-project/curio/lib/paths" -) - -var output = flag.String("output", "", "The output file name, or empty to use stdout.") - -func main() { - flag.Parse() - - its := []struct { - sym string - typ reflect.Type - }{ - - {"SectorIndex", reflect.TypeOf((*pkg_.SectorIndex)(nil)).Elem()}, - } - pkg := &model.Package{ - // NOTE: This behaves contrary to documented behaviour if the - // package name is not the final component of the import path. - // The reflect package doesn't expose the package name, though. - Name: path.Base("github.com/filecoin-project/curio/lib/paths"), - } - - for _, it := range its { - intf, err := model.InterfaceFromInterfaceType(it.typ) - if err != nil { - fmt.Fprintf(os.Stderr, "Reflection: %v\n", err) - os.Exit(1) - } - intf.Name = it.sym - pkg.Interfaces = append(pkg.Interfaces, intf) - } - - outfile := os.Stdout - if len(*output) != 0 { - var err error - outfile, err = os.Create(*output) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to open output file %q", *output) - } - defer func() { - if err := outfile.Close(); err != nil { - fmt.Fprintf(os.Stderr, "failed to close output file %q", *output) - os.Exit(1) - } - }() - } - - if err := gob.NewEncoder(outfile).Encode(pkg); err != nil { - fmt.Fprintf(os.Stderr, "gob encode: %v\n", err) - os.Exit(1) - } -} diff --git a/lib/paths/index.go b/lib/paths/index.go index 35e667fe1..e2d0b2651 100644 --- a/lib/paths/index.go +++ b/lib/paths/index.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/storage/sealer/fsutil" ) diff --git a/lib/paths/local.go b/lib/paths/local.go index 18fd64042..0865a1f19 100644 --- a/lib/paths/local.go +++ b/lib/paths/local.go @@ -337,7 +337,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error { return xerrors.Errorf("declaring storage in index: %w", err) } - if err := st.declareSectors(ctx, p, meta.ID, meta.CanStore, false); err != nil { + if err := st.declareSectors(ctx, p, meta.ID, meta.CanStore, true); err != nil { return err } @@ -385,11 +385,28 @@ func (st *Local) open(ctx context.Context) error { go st.reportHealth(ctx) + go st.startPeriodicRedeclare(ctx) + return nil } var declareCounter atomic.Int32 +func (st *Local) startPeriodicRedeclare(ctx context.Context) { + ticker := time.NewTicker(time.Hour * 4) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := st.Redeclare(ctx, nil, true); err != nil { + log.Errorf("redeclaring storage: %w", err) + } + case <-ctx.Done(): + return + } + } +} + func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMissingDecls bool) error { st.localLk.Lock() defer st.localLk.Unlock() diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 4239d4f60..d0e37a170 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + "runtime" + "runtime/debug" "sync/atomic" "github.com/ethereum/go-ethereum/ethclient" @@ -71,7 +73,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } } - return &MK20{ + ret := &MK20{ miners: miners, db: db, api: mapi, @@ -82,10 +84,22 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI as: as, stor: stor, maxParallelUploads: new(atomic.Int64), - }, nil + } + + go ret.MarkChunkComplete(ctx) + return ret, nil } func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + log.Errorf("panic occurred: %v\n%s", r, trace[:n]) + debug.PrintStack() + } + }() + // Validate the DataSource code, err := deal.Validate(m.db, &m.cfg.Market.StorageMarketConfig.MK20) if err != nil { diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index 2f1a9092d..248e21998 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -2,6 +2,7 @@ package mk20 import ( "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -12,6 +13,9 @@ import ( "strings" "time" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" @@ -42,9 +46,9 @@ func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.Resp err = m.db.QueryRow(ctx, `SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE complete) AS complete, - COUNT(*) FILTER (WHERE NOT complete) AS missing, + COUNT(*) FILTER (WHERE ref_id IS NULL) AS missing, ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE complete) AS completed_chunks, - ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE NOT complete) AS incomplete_chunks + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE ref_id IS NULL) AS incomplete_chunks FROM market_mk20_deal_chunk WHERE @@ -210,11 +214,12 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } var chunkDetails []struct { - Chunk int `db:"chunk"` - Size int64 `db:"chunk_size"` - Complete bool `db:"complete"` + Chunk int `db:"chunk"` + Size int64 `db:"chunk_size"` + Complete bool `db:"complete"` + RefID sql.NullInt64 `db:"ref_id"` } - err := m.db.Select(ctx, &chunkDetails, `SELECT chunk, chunk_size, complete + err := m.db.Select(ctx, &chunkDetails, `SELECT chunk, chunk_size, ref_id, complete FROM market_mk20_deal_chunk WHERE id = $1 AND chunk = $2`, id.String(), chunk) if err != nil { @@ -238,6 +243,11 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w return } + if chunkDetails[0].RefID.Valid { + http.Error(w, "chunk already uploaded", http.StatusConflict) + return + } + defer func() { m.maxParallelUploads.Add(-1) }() @@ -247,12 +257,16 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w chunkSize := chunkDetails[0].Size reader := NewTimeoutReader(data, time.Second*5) m.maxParallelUploads.Add(1) + wr := new(commp.Calc) + + failed := true // Function to write data into StashStore and calculate commP writeFunc := func(f *os.File) error { limitedReader := io.LimitReader(reader, chunkSize+1) // +1 to detect exceeding the limit + writer := io.MultiWriter(f, wr) - size, err := io.CopyBuffer(f, limitedReader, make([]byte, 4<<20)) + size, err := io.CopyBuffer(writer, limitedReader, make([]byte, 4<<20)) if err != nil { return fmt.Errorf("failed to read and write chunk data: %w", err) } @@ -286,7 +300,32 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } } - log.Debugw("uploaded chunk", "deal", id, "chunk", chunk, "stashID", stashID.String()) + defer func() { + if failed { + err = m.stor.StashRemove(ctx, stashID) + if err != nil { + log.Errorw("Failed to remove stash file", "Deal", id, "error", err) + } + } + }() + + digest, pieceSize, err := wr.Digest() + if err != nil { + log.Errorw("failed to calculate commP", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + pcid, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + log.Errorw("failed to calculate piece CID", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + pSize := abi.PaddedPieceSize(pieceSize) + + log.Debugw("uploaded chunk to stash store", "deal", id, "chunk", chunk, "stashID", stashID.String()) stashUrl, err := m.stor.StashURL(stashID) if err != nil { @@ -298,32 +337,62 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w stashUrl.Scheme = dealdata.CustoreScheme log.Debugw("uploading chunk generated URL", "deal", id, "chunk", chunk, "url", stashUrl.String()) + comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var pieceID int64 + err = tx.QueryRow(`SELECT id FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND piece_raw_size = $3`, pcid.String(), pSize, chunkSize).Scan(&pieceID) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, FALSE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id`, pcid.String(), int64(pieceSize), chunkSize).Scan(&pieceID) + if err != nil { + return false, xerrors.Errorf("inserting new parked piece and getting id: %w", err) + } + } else { + return false, xerrors.Errorf("checking existing parked piece: %w", err) + } + } + + // Add parked_piece_ref + var refID int64 + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, FALSE) RETURNING ref_id`, pieceID, stashUrl.String()).Scan(&refID) + if err != nil { + return false, xerrors.Errorf("inserting parked piece ref: %w", err) + } - n, err := m.db.Exec(ctx, `UPDATE market_mk20_deal_chunk SET complete = TRUE, - url = $1 + n, err := tx.Exec(`UPDATE market_mk20_deal_chunk SET ref_id = $1 WHERE id = $2 AND chunk = $3 - AND complete = FALSE`, stashUrl.String(), id.String(), chunk) + AND complete = FALSE + AND ref_id IS NULL`, refID, id.String(), chunk) + if err != nil { + return false, xerrors.Errorf("updating chunk url: %w", err) + } + + return n == 1, nil + }) + if err != nil { log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) http.Error(w, "", http.StatusInternalServerError) - err = m.stor.StashRemove(ctx, stashID) - if err != nil { - log.Errorw("Failed to remove stash file", "Deal", id, "error", err) - } return } - if n == 0 { - log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) + if !comm { + log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", "failed to commit transaction") http.Error(w, "", http.StatusInternalServerError) - err = m.stor.StashRemove(ctx, stashID) - if err != nil { - log.Errorw("Failed to remove stash file", "Deal", id, "error", err) - } return } + failed = false + } func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { @@ -360,3 +429,64 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { w.WriteHeader(http.StatusOK) } + +func (m *MK20) MarkChunkComplete(ctx context.Context) { + ticker := time.NewTicker(time.Second * 3) + defer ticker.Stop() + for { + select { + case <-ticker.C: + markChunksComplete(ctx, m.db) + case <-ctx.Done(): + return + } + } +} + +func markChunksComplete(ctx context.Context, db *harmonydb.DB) { + var chunks []struct { + ID string `db:"id"` + Chunk int `db:"chunk"` + ChunkSize int64 `db:"chunk_size"` + Complete bool `db:"complete"` + RefId int64 `db:"ref_id"` + } + + err := db.Select(ctx, &chunks, `SELECT id, + chunk, + chunk_size, + ref_id, + complete + FROM market_mk20_deal_chunk + WHERE finalize = FALSE + AND complete = FALSE + AND ref_id IS NOT NULL`) + if err != nil { + log.Errorw("failed to get chunks to mark complete", "error", err) + return + } + for _, chunk := range chunks { + var complete bool + err := db.QueryRow(ctx, `SELECT p.complete + FROM parked_pieces AS p + JOIN parked_piece_refs AS r + ON r.piece_id = p.id + WHERE r.ref_id = $1`, chunk.RefId).Scan(&complete) + if err != nil { + log.Errorw("failed to get piece complete status", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) + continue + } + if complete { + _, err := db.Exec(ctx, `UPDATE market_mk20_deal_chunk + SET complete = TRUE + WHERE id = $1 + AND chunk = $2 + AND ref_id = $3 + AND finalize = FALSE`, chunk.ID, chunk.Chunk, chunk.RefId) + if err != nil { + log.Errorw("failed to mark chunk complete", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) + continue + } + } + } +} diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index 73063adf0..b0699de68 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -5,10 +5,11 @@ import ( "errors" "fmt" "io" - "net/url" "os" "time" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/storiface" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/oklog/ulid" @@ -33,13 +34,15 @@ type AggregateChunksTask struct { db *harmonydb.DB stor paths.StashStore remote *paths.Remote + sc *ffi.SealCalls } -func NewAggregateChunksTask(db *harmonydb.DB, stor paths.StashStore, remote *paths.Remote) *AggregateChunksTask { +func NewAggregateChunksTask(db *harmonydb.DB, stor paths.StashStore, remote *paths.Remote, sc *ffi.SealCalls) *AggregateChunksTask { return &AggregateChunksTask{ db: db, stor: stor, remote: remote, + sc: sc, } } @@ -50,7 +53,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo ID string `db:"id"` Chunk int `db:"chunk"` Size int64 `db:"chunk_size"` - URL string `db:"url"` + RefID int64 `db:"ref_id"` } err = a.db.Select(ctx, &chunks, ` @@ -58,7 +61,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo id, chunk, chunk_size, - url + ref_id FROM market_mk20_deal_chunk WHERE @@ -112,21 +115,34 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } var readers []io.Reader + var refIds []int64 for _, chunk := range chunks { - goUrl, err := url.Parse(chunk.URL) + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = a.db.Select(ctx, &pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, chunk.RefID) if err != nil { - return false, xerrors.Errorf("parsing data URL: %w", err) + return false, xerrors.Errorf("getting pieceID: %w", err) + } + + if len(pieceID) != 1 { + return false, xerrors.Errorf("expected 1 pieceID, got %d", len(pieceID)) } - upr := dealdata.NewUrlReader(a.remote, goUrl.String(), nil, chunk.Size) + pr, err := a.sc.PieceReader(ctx, pieceID[0].PieceID) + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } - reader := upr + reader := pr defer func() { - _ = upr.Close() + _ = pr.Close() }() readers = append(readers, reader) + refIds = append(refIds, chunk.RefID) } rd := io.MultiReader(readers...) @@ -250,6 +266,11 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo if err != nil { return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) } + + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) + if err != nil { + return false, xerrors.Errorf("deleting parked piece refs: %w", err) + } } else { return false, xerrors.Errorf("not implemented for PDP") // TODO: Do what is required for PDP @@ -280,7 +301,7 @@ func (a *AggregateChunksTask) TypeDetails() harmonytask.TaskTypeDetails { Ram: 4 << 30, }, MaxFailures: 3, - IAmBored: passcall.Every(30*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { return a.schedule(context.Background(), taskFunc) }), } @@ -302,7 +323,7 @@ func (a *AggregateChunksTask) schedule(ctx context.Context, taskFunc harmonytask WHERE complete = TRUE AND finalize = TRUE AND finalize_task_id IS NULL - AND url IS NOT NULL + AND ref_id IS NOT NULL ) ORDER BY id LIMIT 1;`).Scan(&mid, &count) @@ -321,7 +342,7 @@ func (a *AggregateChunksTask) schedule(ctx context.Context, taskFunc harmonytask AND complete = TRUE AND finalize = TRUE AND finalize_task_id IS NULL - AND url IS NOT NULL`, id, mid) + AND ref_id IS NOT NULL`, id, mid) if err != nil { return false, xerrors.Errorf("updating chunk finalize task: %w", err) } diff --git a/tasks/piece/task_park_piece.go b/tasks/piece/task_park_piece.go index d8c67e896..08f8a6175 100644 --- a/tasks/piece/task_park_piece.go +++ b/tasks/piece/task_park_piece.go @@ -41,8 +41,8 @@ type ParkPieceTask struct { longTerm bool // Indicates if the task is for long-term pieces } -func NewParkPieceTask(db *harmonydb.DB, sc *ffi2.SealCalls, max int) (*ParkPieceTask, error) { - return newPieceTask(db, sc, nil, max, false) +func NewParkPieceTask(db *harmonydb.DB, sc *ffi2.SealCalls, remote *paths.Remote, max int) (*ParkPieceTask, error) { + return newPieceTask(db, sc, remote, max, false) } func NewStorePieceTask(db *harmonydb.DB, sc *ffi2.SealCalls, remote *paths.Remote, max int) (*ParkPieceTask, error) { diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index 2ea4ebafa..b607b8e54 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -806,3 +806,41 @@ func (a *WebRPC) ListDataSources(ctx context.Context) (map[string]bool, error) { } return datasourceMap, nil } + +type UploadStatus struct { + ID string `json:"id"` + Status mk20.UploadStatus `json:"status"` +} + +func (a *WebRPC) ChunkUploadStatus(ctx context.Context, idStr string) (*UploadStatus, error) { + id, err := ulid.Parse(idStr) + if err != nil { + return nil, fmt.Errorf("invalid chunk upload id: %w", err) + } + + var status mk20.UploadStatus + + err = a.deps.DB.QueryRow(ctx, `SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE complete) AS complete, + COUNT(*) FILTER (WHERE NOT complete) AS missing, + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE complete) AS completed_chunks, + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE NOT complete) AS incomplete_chunks + FROM + market_mk20_deal_chunk + WHERE + id = $1 + GROUP BY + id;`, id.String()).Scan(&status.TotalChunks, &status.Uploaded, &status.Missing, &status.UploadedChunks, &status.MissingChunks) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return nil, xerrors.Errorf("failed to get chunk upload status: %w", err) + } + return nil, nil + } + + return &UploadStatus{ + ID: idStr, + Status: status, + }, nil +} diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 87b0ff77c..5523f9e5b 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -58,7 +58,7 @@ class DealDetails extends LitElement { Details - ${this.renderDataSource(data)} + ${this.renderDataSource(data, identifier)} @@ -66,7 +66,7 @@ class DealDetails extends LitElement { `; } - renderDataSource(data){ + renderDataSource(data, id){ if (!data) return ''; if (data.source_http) { return html` @@ -95,7 +95,7 @@ class DealDetails extends LitElement { if (data.source_httpput) { return html` - HTTP Put + HTTP Put ${data?.source_httpput ? this.renderSourceHttpPut(data.source_httpput) : ''} ` diff --git a/web/static/pages/upload-status/index.html b/web/static/pages/upload-status/index.html new file mode 100644 index 000000000..c99099b27 --- /dev/null +++ b/web/static/pages/upload-status/index.html @@ -0,0 +1,31 @@ + + + + + Deals + + + + + + +
    +
    +
    +
    +

    Upload Status

    +
    +
    +
    +
    +
    +
    + +
    +
    +
    +
    +     + + + \ No newline at end of file diff --git a/web/static/pages/upload-status/status.mjs b/web/static/pages/upload-status/status.mjs new file mode 100644 index 000000000..b61c52fab --- /dev/null +++ b/web/static/pages/upload-status/status.mjs @@ -0,0 +1,94 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; + +class UploadStatus extends LitElement { + constructor() { + super(); + this.loaddata(); + } + + static styles = css` + .chunk-box { + display: grid; + grid-template-columns: repeat(16, auto); + grid-template-rows: repeat(3, auto); + grid-gap: 1px; + } + .chunk-entry { + width: 10px; + height: 10px; + background-color: grey; + margin: 1px; + } + .chunk-complete { + background-color: green; + } + .chunk-missing + background-color: red; + } + ` + + async loaddata() { + try { + const params = new URLSearchParams(window.location.search); + this.data = await RPCCall('ChunkUploadStatus', [params.get('id')]); + this.requestUpdate(); + } catch (error) { + console.error('Failed to load upload status:', error); + alert(`Failed to load upload status: ${error.message}`); + } + } + + render() { + if (!this.data) return html`

    No data.

    `; + + return html` + + + + + + + + + +
    Identifier${this.data.id}
    Total Chunks${this.data.status.total_chunks}
    Uploaded${this.data.status.uploaded}
    Missing${this.data.status.missing}
    Status${this.renderChunks(this.data.status)}
    + `; + } + + renderChunks(status) { + const totalChunks = status.total_chunks; + const missingChunks = status.missing_chunks || []; + const uploadedChunksSet = new Set(); + + if (status.uploaded_chunks) { + status.uploaded_chunks.forEach(chunk => uploadedChunksSet.add(chunk)); + } + + // Create chunk entries + const chunkEntries = Array.from({ length: totalChunks }, (_, i) => { + const chunkIndex = i + 1; // Chunks start from 1 + const isMissing = missingChunks.includes(chunkIndex); + const isUploaded = uploadedChunksSet.has(chunkIndex); + + return html` +
    +
    + `; + }); + + return html` +
    + ${chunkEntries} +
    + `; + } +} +customElements.define('upload-status', UploadStatus); + From 4b8da8ec5b50a64a069c6fb6a853545deb4fdd4c Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 26 Jun 2025 20:46:11 +0400 Subject: [PATCH 17/55] pcid2, pdpv1, retrievalv1 --- cmd/curio/tasks/tasks.go | 4 +- cmd/sptool/toolbox_deal_client.go | 151 +++--- .../piece-server/sample/mk20-aggregate-car.sh | 10 +- .../piece-server/sample/mk20-random-deal.sh | 14 +- documentation/en/curio-cli/sptool.md | 19 +- .../harmonydb/sql/20250505-market_mk20.sql | 85 ++- lib/cachedreader/cachedreader.go | 4 +- lib/commcidv2/commcidv2.go | 59 +++ lib/ffi/piece_funcs.go | 74 ++- lib/testutils/testutils.go | 71 +-- market/mk12/mk12.go | 6 +- market/mk20/ddo_v1.go | 19 +- market/mk20/http/info.md | 65 ++- market/mk20/mk20.go | 49 +- market/mk20/mk20_upload.go | 287 +++++----- market/mk20/pdp_v1.go | 28 + market/mk20/retrieval_v1.go | 38 ++ market/mk20/types.go | 55 +- market/mk20/utils.go | 491 +++++++++--------- tasks/indexing/task_check_indexes.go | 123 +++-- tasks/indexing/task_indexing.go | 114 ++-- tasks/indexing/task_ipni.go | 18 +- tasks/pdp/task_addroot.go | 190 +++++++ tasks/piece/task_aggregate_chunks.go | 182 ++++--- tasks/piece/task_park_piece.go | 3 +- tasks/storage-market/mk20.go | 56 +- tasks/storage-market/storage_market.go | 62 ++- tasks/storage-market/task_aggregation.go | 115 ++-- tasks/storage-market/task_commp.go | 10 +- web/api/webrpc/deals.go | 7 +- web/api/webrpc/ipni.go | 11 +- web/api/webrpc/market.go | 384 +++++++------- web/api/webrpc/market_20.go | 109 ++-- web/api/webrpc/pdp.go | 2 +- web/api/webrpc/sector.go | 15 +- .../pages/market-settings/allow-list.mjs | 1 - web/static/pages/mk12-deal/deal.mjs | 8 +- .../pages/mk12-deals/deal-pipelines.mjs | 7 +- web/static/pages/mk12-deals/index.html | 8 - web/static/pages/mk12-deals/mk12-deals.mjs | 19 +- web/static/pages/mk12-deals/mk12ddo-list.mjs | 29 +- web/static/pages/mk20-deal/deal.mjs | 31 +- web/static/pages/mk20/ddo-pipeline.mjs | 6 +- web/static/pages/mk20/ddo.mjs | 17 +- web/static/pages/piece/piece-info.mjs | 4 +- web/static/pages/sector/sector-info.mjs | 8 +- 46 files changed, 1837 insertions(+), 1231 deletions(-) create mode 100644 market/mk20/pdp_v1.go create mode 100644 market/mk20/retrieval_v1.go create mode 100644 tasks/pdp/task_addroot.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index ead515eae..fb56a31e7 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -225,7 +225,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan return nil, err } cleanupPieceTask := piece2.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0) - aggregateChunksTask := piece2.NewAggregateChunksTask(db, lstor, stor, must.One(slrLazy.Val())) + aggregateChunksTask := piece2.NewAggregateChunksTask(db, stor, must.One(slrLazy.Val())) activeTasks = append(activeTasks, parkPieceTask, cleanupPieceTask, aggregateChunksTask) } } @@ -249,7 +249,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan var dm *storage_market.CurioStorageDealMarket if cfg.Subsystems.EnableDealMarket { // Main market poller should run on all nodes - dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as, lstor) + dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as, must.One(slrLazy.Val())) err := dm.StartMarket(ctx) if err != nil { return nil, err diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 29318f02c..83cb30764 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -15,7 +15,6 @@ import ( "os" "os/signal" "path/filepath" - "strconv" "strings" "syscall" "time" @@ -23,7 +22,6 @@ import ( "github.com/dustin/go-humanize" "github.com/google/uuid" "github.com/ipfs/go-cid" - "github.com/ipfs/go-cidutil/cidenc" "github.com/ipni/go-libipni/maurl" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/crypto" @@ -32,7 +30,6 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/mitchellh/go-homedir" "github.com/multiformats/go-multiaddr" - "github.com/multiformats/go-multibase" "github.com/oklog/ulid" "github.com/urfave/cli/v2" "golang.org/x/exp/mmap" @@ -41,18 +38,20 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/keystore" "github.com/filecoin-project/curio/lib/testutils" mk12_libp2p "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/mk12" "github.com/filecoin-project/curio/market/mk20" - "github.com/filecoin-project/lotus/api" + lapi "github.com/filecoin-project/lotus/api" chain_types "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" lcli "github.com/filecoin-project/lotus/cli" @@ -613,7 +612,7 @@ func dealProposal(ctx context.Context, n *Node, clientAddr address.Address, root return nil, err } - sig, err := n.Wallet.WalletSign(ctx, clientAddr, buf, api.MsgMeta{Type: api.MTDealProposal}) + sig, err := n.Wallet.WalletSign(ctx, clientAddr, buf, lapi.MsgMeta{Type: lapi.MTDealProposal}) if err != nil { return nil, xerrors.Errorf("wallet sign failed: %w", err) } @@ -1305,7 +1304,7 @@ var walletSign = &cli.Command{ return err } - sig, err := n.Wallet.WalletSign(ctx, addr, msg, api.MsgMeta{Type: api.MTUnknown}) + sig, err := n.Wallet.WalletSign(ctx, addr, msg, lapi.MsgMeta{Type: lapi.MTUnknown}) if err != nil { return err } @@ -1416,7 +1415,7 @@ var dealStatusCmd = &cli.Command{ return fmt.Errorf("getting uuid bytes: %w", err) } - sig, err := n.Wallet.WalletSign(ctx, walletAddr, uuidBytes, api.MsgMeta{Type: api.MTDealProposal}) + sig, err := n.Wallet.WalletSign(ctx, walletAddr, uuidBytes, lapi.MsgMeta{Type: lapi.MTDealProposal}) if err != nil { return fmt.Errorf("signing uuid bytes: %w", err) } @@ -1579,12 +1578,57 @@ var mk20Clientcmd = &cli.Command{ }, Subcommands: []*cli.Command{ initCmd, + comm2Cmd, mk20DealCmd, mk20ClientMakeAggregateCmd, mk20ClientUploadCmd, }, } +var comm2Cmd = &cli.Command{ + Name: "commp", + Usage: "", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() != 1 { + return fmt.Errorf("usage: commP ") + } + + inPath := cctx.Args().Get(0) + + rdr, err := os.Open(inPath) + if err != nil { + return err + } + defer rdr.Close() //nolint:errcheck + + stat, err := os.Stat(inPath) + if err != nil { + return err + } + + wr := new(commp.Calc) + _, err = io.CopyBuffer(wr, rdr, make([]byte, 2<<20)) + if err != nil { + return fmt.Errorf("copy into commp writer: %w", err) + } + + digest, _, err := wr.Digest() + if err != nil { + return fmt.Errorf("generating digest failed: %w", err) + } + + commp, err := commcidv2.NewSha2CommP(uint64(stat.Size()), digest) + if err != nil { + return fmt.Errorf("computing commP failed: %w", err) + } + + fmt.Println("CommP CID: ", commp.PCidV2().String()) + fmt.Println("Car file size: ", stat.Size()) + return nil + }, +} + var mk20DealCmd = &cli.Command{ Name: "deal", Usage: "Make a mk20 deal with Curio", @@ -1597,23 +1641,14 @@ var mk20DealCmd = &cli.Command{ Name: "http-headers", Usage: "http headers to be passed with the request (e.g key=value)", }, - &cli.Uint64Flag{ - Name: "car-size", - Usage: "size of the CAR file: required for online deals", - }, &cli.StringFlag{ Name: "provider", Usage: "storage provider on-chain address", Required: true, }, &cli.StringFlag{ - Name: "commp", - Usage: "commp of the CAR file", - Required: true, - }, - &cli.Uint64Flag{ - Name: "piece-size", - Usage: "size of the CAR file as a padded piece", + Name: "pcidv2", + Usage: "pcidv2 of the CAR file", Required: true, }, &cli.IntFlag{ @@ -1728,19 +1763,12 @@ var mk20DealCmd = &cli.Command{ hurls = append(hurls, hurl) } - commp := cctx.String("commp") + commp := cctx.String("pcidv2") pieceCid, err := cid.Parse(commp) if err != nil { - return xerrors.Errorf("parsing commp '%s': %w", commp, err) + return xerrors.Errorf("parsing pcidv2 '%s': %w", commp, err) } - pieceSize := cctx.Uint64("piece-size") - if pieceSize == 0 { - return xerrors.Errorf("must provide piece-size parameter for CAR url") - } - - carFileSize := cctx.Uint64("car-size") - var headers http.Header for _, header := range cctx.StringSlice("http-headers") { @@ -1756,7 +1784,6 @@ var mk20DealCmd = &cli.Command{ if cctx.IsSet("aggregate") { d = mk20.DataSource{ PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), Format: mk20.PieceDataFormat{ Aggregate: &mk20.FormatAggregate{ Type: mk20.AggregateTypeV1, @@ -1781,10 +1808,10 @@ var mk20DealCmd = &cli.Command{ for scanner.Scan() { line := scanner.Text() parts := strings.Split(line, "\t") - if len(parts) != 4 { - return fmt.Errorf("invalid line format. Expected pieceCid, pieceSize, carSize, url at %s", line) + if len(parts) != 2 { + return fmt.Errorf("invalid line format. Expected pieceCidV2, url at %s", line) } - if parts[0] == "" || parts[1] == "" || parts[2] == "" || parts[3] == "" { + if parts[0] == "" || parts[1] == "" { return fmt.Errorf("empty column value in the input file at %s", line) } @@ -1792,29 +1819,18 @@ var mk20DealCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to parse CID: %w", err) } - pieceSize, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return fmt.Errorf("failed to parse size %w", err) - } - rawSize, err := strconv.ParseInt(parts[2], 10, 64) - if err != nil { - return fmt.Errorf("failed to parse raw size %w", err) - } - - url, err := url.Parse(parts[3]) + url, err := url.Parse(parts[1]) if err != nil { return fmt.Errorf("failed to parse url: %w", err) } pieces = append(pieces, mk20.DataSource{ PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), Format: mk20.PieceDataFormat{ Car: &mk20.FormatCar{}, }, SourceHTTP: &mk20.DataSourceHTTP{ - RawSize: uint64(rawSize), URLs: []mk20.HttpUrl{ { URL: url.String(), @@ -1833,32 +1849,22 @@ var mk20DealCmd = &cli.Command{ Pieces: pieces, } } else { - if carFileSize == 0 { - return xerrors.Errorf("size of car file cannot be 0") - } - if !cctx.IsSet("http-url") { if cctx.Bool("put") { d = mk20.DataSource{ PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), Format: mk20.PieceDataFormat{ Car: &mk20.FormatCar{}, }, - SourceHttpPut: &mk20.DataSourceHttpPut{ - RawSize: carFileSize, - }, + SourceHttpPut: &mk20.DataSourceHttpPut{}, } } else { d = mk20.DataSource{ PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), Format: mk20.PieceDataFormat{ Car: &mk20.FormatCar{}, }, - SourceOffline: &mk20.DataSourceOffline{ - RawSize: carFileSize, - }, + SourceOffline: &mk20.DataSourceOffline{}, } } } else { @@ -1868,12 +1874,10 @@ var mk20DealCmd = &cli.Command{ } d = mk20.DataSource{ PieceCID: pieceCid, - Size: abi.PaddedPieceSize(pieceSize), Format: mk20.PieceDataFormat{ Car: &mk20.FormatCar{}, }, SourceHTTP: &mk20.DataSourceHTTP{ - RawSize: carFileSize, URLs: []mk20.HttpUrl{ { URL: url.String(), @@ -1890,14 +1894,15 @@ var mk20DealCmd = &cli.Command{ p := mk20.Products{ DDOV1: &mk20.DDOV1{ Provider: maddr, - Client: walletAddr, PieceManager: walletAddr, Duration: abi.ChainEpoch(cctx.Int64("duration")), ContractAddress: cctx.String("contract-address"), ContractVerifyMethod: cctx.String("contract-verify-method"), ContractVerifyMethodParams: []byte("test bytes"), - Indexing: cctx.Bool("indexing"), - AnnounceToIPNI: cctx.Bool("announce"), + }, + RetrievalV1: &mk20.RetrievalV1{ + Indexing: cctx.Bool("indexing"), + AnnouncePayload: cctx.Bool("announce"), }, } @@ -1912,9 +1917,26 @@ var mk20DealCmd = &cli.Command{ } log.Debugw("generated deal id", "id", id) + msg, err := id.MarshalBinary() + if err != nil { + return xerrors.Errorf("failed to marshal deal id: %w", err) + } + + sig, err := n.Wallet.WalletSign(ctx, walletAddr, msg, lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return xerrors.Errorf("failed to sign deal proposal: %w", err) + } + + msgb, err := sig.MarshalBinary() + if err != nil { + return xerrors.Errorf("failed to marshal deal proposal signature: %w", err) + } + deal := mk20.Deal{ Identifier: id, - Data: d, + Client: walletAddr, + Signature: msgb, + Data: &d, Products: p, } @@ -1971,19 +1993,18 @@ var mk20ClientMakeAggregateCmd = &cli.Command{ &cli.BoolFlag{ Name: "out", Usage: "output the aggregate file", + Value: true, }, }, Action: func(cctx *cli.Context) error { size := abi.PaddedPieceSize(cctx.Uint64("piece-size")) files := cctx.StringSlice("files") out := cctx.Bool("out") - pcid, size, err := testutils.CreateAggregateFromCars(files, size, out) + pcid, err := testutils.CreateAggregateFromCars(files, size, out) if err != nil { return err } - encoder := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} - fmt.Println("CommP CID: ", encoder.Encode(pcid)) - fmt.Println("Piece size: ", size) + fmt.Println("CommP CID: ", pcid.String()) return nil }, } diff --git a/docker/piece-server/sample/mk20-aggregate-car.sh b/docker/piece-server/sample/mk20-aggregate-car.sh index e95f6d2bf..ae2935415 100755 --- a/docker/piece-server/sample/mk20-aggregate-car.sh +++ b/docker/piece-server/sample/mk20-aggregate-car.sh @@ -59,11 +59,10 @@ echo "$aggregate_output" # Step 3: Extract `CommP CID` and `Piece size` from the aggregate output commp_cid=$(echo "$aggregate_output" | awk -F': ' '/CommP CID/ {print $2}' | xargs) -piece_size=$(echo "$aggregate_output" | awk -F': ' '/Piece size/ {print $2}' | xargs) # Validate that we got proper output -if [[ -z "$commp_cid" || -z "$piece_size" ]]; then - echo "Error: Failed to extract CommP CID or Piece size from aggregation output" >&2 +if [[ -z "$commp_cid" ]]; then + echo "Error: Failed to extract CommP CID from aggregation output" >&2 exit 1 fi @@ -80,16 +79,15 @@ fi # Step 5: Print Results echo -e "\n${ci}Aggregation Results:${cn}" echo "CommP CID: $commp_cid" -echo "Piece Size: $piece_size" miner_actor=$(lotus state list-miners | grep -v t01000) ################################################################################### printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ ---commp=$commp_cid --piece-size=$piece_size --contract-address 0xtest --contract-verify-method test \ +--pcidv2=$commp_cid --contract-address 0xtest --contract-verify-method test \ --aggregate "$aggregate_file"\n\n${cn}" -sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$commp_cid --piece-size=$piece_size --contract-address 0xtest --contract-verify-method test --aggregate "$aggregate_file" +sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --pcidv2=$commp_cid --contract-address 0xtest --contract-verify-method test --aggregate "$aggregate_file" echo -e "\nDone!" \ No newline at end of file diff --git a/docker/piece-server/sample/mk20-random-deal.sh b/docker/piece-server/sample/mk20-random-deal.sh index 6f8aa6028..51a43f4ce 100755 --- a/docker/piece-server/sample/mk20-random-deal.sh +++ b/docker/piece-server/sample/mk20-random-deal.sh @@ -12,7 +12,7 @@ links="${4:-100}" printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` -read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk12-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') +read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk20-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') mv $FILE /var/lib/curio-client/data/$COMMP_CID @@ -21,10 +21,10 @@ miner_actor=$(lotus state list-miners | grep -v t01000) if [ "$put" == "true" ]; then ################################################################################### printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ - --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --pcidv2=$COMMP_CID \ --contract-address 0xtest --contract-verify-method test --put\n\n${cn}" - sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test --put + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test --put else @@ -32,19 +32,19 @@ else ################################################################################### printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ - --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --pcidv2=$COMMP_CID \ --contract-address 0xtest --contract-verify-method test\n\n${cn}" - sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test else ################################################################################### printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ --http-url=http://piece-server:12320/pieces?id=$COMMP_CID \ - --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ + --pcidv2=$COMMP_CID \ --contract-address 0xtest --contract-verify-method test\n\n${cn}" - sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --contract-address 0xtest --contract-verify-method test + sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test fi diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 6a643eb06..a957db255 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -900,6 +900,7 @@ USAGE: COMMANDS: init Initialise curio mk12 client repo + commp deal Make a mk20 deal with Curio aggregate Create a new aggregate from a list of CAR files upload Upload a file to the storage provider @@ -922,6 +923,18 @@ OPTIONS: --help, -h show help ``` +#### sptool toolbox mk20-client commp +``` +NAME: + sptool toolbox mk20-client commp + +USAGE: + sptool toolbox mk20-client commp [command options] + +OPTIONS: + --help, -h show help +``` + #### sptool toolbox mk20-client deal ``` NAME: @@ -933,10 +946,8 @@ USAGE: OPTIONS: --http-url value http url to CAR file --http-headers value [ --http-headers value ] http headers to be passed with the request (e.g key=value) - --car-size value size of the CAR file: required for online deals (default: 0) --provider value storage provider on-chain address - --commp value commp of the CAR file - --piece-size value size of the CAR file as a padded piece (default: 0) + --pcidv2 value pcidv2 of the CAR file --duration value duration of the deal in epochs (default: 518400) --contract-address value contract address of the deal --contract-verify-method value contract verify method of the deal @@ -960,7 +971,7 @@ USAGE: OPTIONS: --files value [ --files value ] list of CAR files to aggregate --piece-size value piece size of the aggregate (default: 0) - --out output the aggregate file (default: false) + --out output the aggregate file (default: true) --help, -h show help ``` diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index a425876bc..3a5c50861 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -1,3 +1,11 @@ +-- Add raw_size column to mk12 deals to calculate pieceCidV2 +ALTER TABLE market_mk12_deals + ADD COLUMN raw_size BIGINT; + +-- Add raw_size column to mk12-ddo deals to calculate pieceCidV2 +ALTER TABLE market_direct_deals + ADD COLUMN raw_size BIGINT; + -- Drop the existing primary key constraint for market_piece_metadata ALTER TABLE market_piece_metadata DROP CONSTRAINT market_piece_metadata_pkey; @@ -113,8 +121,36 @@ BEGIN -- If all conditions are met, insert the new task into ipni_task INSERT INTO ipni_task (sp_id, id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) VALUES (_sp_id, _id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); - END; - $$ LANGUAGE plpgsql; +END; +$$ LANGUAGE plpgsql; + + +-- Update raw_size for existing deals (One time backfill migration) +BEGIN; + UPDATE market_mk12_deals d + SET raw_size = mpd.raw_size + FROM market_piece_deal mpd + WHERE d.uuid = mpd.id; + + UPDATE market_direct_deals d + SET raw_size = mpd.raw_size + FROM market_piece_deal mpd + WHERE d.uuid = mpd.id; + + UPDATE market_mk12_deals d + SET raw_size = p.raw_size + FROM market_mk12_deal_pipeline p + WHERE d.uuid = p.uuid + AND d.raw_size IS NULL + AND p.raw_size IS NOT NULL; + + UPDATE market_direct_deals d + SET raw_size = p.raw_size + FROM market_mk12_deal_pipeline p + WHERE d.uuid = p.uuid + AND d.raw_size IS NULL + AND p.raw_size IS NOT NULL; +COMMIT; CREATE TABLE ddo_contracts ( @@ -124,23 +160,18 @@ CREATE TABLE ddo_contracts ( CREATE TABLE market_mk20_deal ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), - - sp_id BIGINT NOT NULL, - id TEXT PRIMARY KEY, - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, + client TEXT NOT NULL, + piece_cid_v2 TEXT, + piece_cid TEXT, -- This is pieceCid V1 to allow easy table lookups + piece_size BIGINT, + raw_size BIGINT, -- For ease - format JSONB NOT NULL, - source_http JSONB NOT NULL DEFAULT 'null', - source_aggregate JSONB NOT NULL DEFAULT 'null', - source_offline JSONB NOT NULL DEFAULT 'null', - source_http_put JSONB NOT NULL DEFAULT 'null', + data JSONB NOT NULL DEFAULT 'null', ddo_v1 JSONB NOT NULL DEFAULT 'null', - market_deal_id TEXT DEFAULT NULL, - - error TEXT DEFAULT NULL + retrieval_v1 JSONB NOT NULL DEFAULT 'null', + pdp_v1 JSONB NOT NULL DEFAULT 'null' ); CREATE TABLE market_mk20_pipeline ( @@ -149,7 +180,8 @@ CREATE TABLE market_mk20_pipeline ( sp_id BIGINT NOT NULL, contract TEXT NOT NULL, client TEXT NOT NULL, - piece_cid TEXT NOT NULL, + piece_cid_v2 TEXT NOT NULL, + piece_cid TEXT NOT NULL, -- This is pieceCid V1 to allow easy table lookups piece_size BIGINT NOT NULL, raw_size BIGINT NOT NULL, offline BOOLEAN NOT NULL, @@ -194,7 +226,7 @@ CREATE TABLE market_mk20_pipeline_waiting ( CREATE TABLE market_mk20_download_pipeline ( id TEXT NOT NULL, - piece_cid TEXT NOT NULL, + piece_cid TEXT NOT NULL, -- This is pieceCid V1 to allow easy table lookups piece_size BIGINT NOT NULL, ref_ids BIGINT[] NOT NULL, PRIMARY KEY (id, piece_cid, piece_size) @@ -232,6 +264,8 @@ CREATE TABLE market_mk20_data_source ( ); INSERT INTO market_mk20_products (name, enabled) VALUES ('ddo_v1', TRUE); +INSERT INTO market_mk20_products (name, enabled) VALUES ('retrieval_v1', TRUE); +INSERT INTO market_mk20_products (name, enabled) VALUES ('pdp_v1', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('http', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); @@ -243,7 +277,7 @@ CREATE OR REPLACE FUNCTION process_offline_download( _piece_size BIGINT ) RETURNS BOOLEAN AS $$ DECLARE -_url TEXT; + _url TEXT; _headers JSONB; _raw_size BIGINT; _deal_aggregation INT; @@ -305,9 +339,24 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- Add column to skip scheduling piece_park +ALTER TABLE parked_pieces + ADD COLUMN skip BOOLEAN DEFAULT FALSE; +CREATE TABLE pdp_pipeline ( + id TEXT PRIMARY KEY, + piece_cid TEXT NOT NULL, -- v2 piece_cid + + add_root_task_id BIGINT DEFAULT NULL, + after_add_root BOOLEAN DEFAULT FALSE, + indexing BOOLEAN DEFAULT FALSE, + indexing_created_at TIMESTAMPTZ DEFAULT NULL, + indexing_task_id BIGINT DEFAULT NULL, + indexed BOOLEAN DEFAULT FALSE, + complete BOOLEAN DEFAULT FALSE +); diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index 88976bab1..b42976ad4 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -334,13 +334,13 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid reader, size, err := cpr.getPieceReaderFromAggregate(readerCtx, pieceCidV2) if err != nil { - log.Warnw("failed to get piece reader from aggregate", "piececid", pieceCidV2.String(), "err", err) + log.Debugw("failed to get piece reader from aggregate", "piececid", pieceCidV2.String(), "err", err) aerr := err reader, size, err = cpr.getPieceReaderFromSector(readerCtx, pieceCidV2) if err != nil { - log.Warnw("failed to get piece reader from sector", "piececid", pieceCidV2.String(), "err", err) + log.Debugw("failed to get piece reader from sector", "piececid", pieceCidV2.String(), "err", err) serr := err // Try getPieceReaderFromPiecePark reader, size, err = cpr.getPieceReaderFromPiecePark(readerCtx, pieceCidV2) diff --git a/lib/commcidv2/commcidv2.go b/lib/commcidv2/commcidv2.go index 38b131fc7..aa804fa7e 100644 --- a/lib/commcidv2/commcidv2.go +++ b/lib/commcidv2/commcidv2.go @@ -5,6 +5,7 @@ import ( "github.com/ipfs/go-cid" pool "github.com/libp2p/go-buffer-pool" + "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/multiformats/go-varint" "golang.org/x/xerrors" @@ -174,3 +175,61 @@ func (cp *CommP) PCidV2() cid.Cid { } func (cp *CommP) Digest() []byte { return cp.digest } + +func PieceCidV2FromV1(v1PieceCid cid.Cid, payloadsize uint64) (cid.Cid, error) { + decoded, err := multihash.Decode(v1PieceCid.Hash()) + if err != nil { + return cid.Undef, xerrors.Errorf("Error decoding data commitment hash: %w", err) + } + + filCodec := multicodec.Code(v1PieceCid.Type()) + filMh := multicodec.Code(decoded.Code) + + switch filCodec { + case multicodec.FilCommitmentUnsealed: + if filMh != multicodec.Sha2_256Trunc254Padded { + return cid.Undef, xerrors.Errorf("unexpected hash: %d", filMh) + } + case multicodec.FilCommitmentSealed: + if filMh != multicodec.PoseidonBls12_381A2Fc1 { + return cid.Undef, xerrors.Errorf("unexpected hash: %d", filMh) + } + default: // neither of the codecs above: we are not in Fil teritory + return cid.Undef, xerrors.Errorf("unexpected codec: %d", filCodec) + } + + if len(decoded.Digest) != 32 { + return cid.Undef, xerrors.Errorf("commitments must be 32 bytes long") + } + if filCodec != multicodec.FilCommitmentUnsealed { + return cid.Undef, xerrors.Errorf("unexpected codec: %d", filCodec) + } + + c, err := NewSha2CommP(payloadsize, decoded.Digest) + if err != nil { + return cid.Undef, xerrors.Errorf("error creating CommP: %w", err) + } + + return c.PCidV2(), nil +} + +func IsPieceCidV2(c cid.Cid) bool { + if c.Type() != uint64(multicodec.Raw) { + return false + } + + decoded, err := multihash.Decode(c.Hash()) + if err != nil { + return false + } + + if decoded.Code != uint64(multicodec.Fr32Sha256Trunc254Padbintree) { + return false + } + + if len(decoded.Digest) < 34 { + return false + } + + return true +} diff --git a/lib/ffi/piece_funcs.go b/lib/ffi/piece_funcs.go index c719b1a12..b01625c6a 100644 --- a/lib/ffi/piece_funcs.go +++ b/lib/ffi/piece_funcs.go @@ -8,8 +8,12 @@ import ( "golang.org/x/xerrors" + commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/harmony/harmonytask" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) func (sb *SealCalls) WritePiece(ctx context.Context, taskID *harmonytask.TaskID, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType) error { @@ -73,3 +77,71 @@ func (sb *SealCalls) PieceReader(ctx context.Context, id storiface.PieceNumber) func (sb *SealCalls) RemovePiece(ctx context.Context, id storiface.PieceNumber) error { return sb.sectors.storage.Remove(ctx, id.Ref().ID, storiface.FTPiece, true, nil) } + +func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType) (abi.PieceInfo, error) { + // Use storageType in AcquireSector + paths, _, done, err := sb.sectors.AcquireSector(ctx, nil, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType) + if err != nil { + return abi.PieceInfo{}, err + } + defer done() + + dest := paths.Piece + tempDest := dest + storiface.TempSuffix + + destFile, err := os.OpenFile(tempDest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("creating temp piece file '%s': %w", tempDest, err) + } + + removeTemp := true + defer func() { + if removeTemp { + rerr := os.Remove(tempDest) + if rerr != nil { + log.Errorf("removing temp file: %+v", rerr) + } + } + }() + + copyStart := time.Now() + + wr := new(commp.Calc) + writers := io.MultiWriter(wr, destFile) + + n, err := io.CopyBuffer(writers, io.LimitReader(data, size), make([]byte, 8<<20)) + if err != nil { + _ = destFile.Close() + return abi.PieceInfo{}, xerrors.Errorf("copying piece data: %w", err) + } + + if err := destFile.Close(); err != nil { + return abi.PieceInfo{}, xerrors.Errorf("closing temp piece file: %w", err) + } + + if n != size { + return abi.PieceInfo{}, xerrors.Errorf("short write: %d", n) + } + + digest, pieceSize, err := wr.Digest() + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("computing piece digest: %w", err) + } + + pcid, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("computing piece CID: %w", err) + } + psize := abi.PaddedPieceSize(pieceSize) + + copyEnd := time.Now() + + log.Infow("wrote piece", "piece", pieceID, "size", size, "duration", copyEnd.Sub(copyStart), "dest", dest, "MiB/s", float64(size)/(1<<20)/copyEnd.Sub(copyStart).Seconds()) + + if err := os.Rename(tempDest, dest); err != nil { + return abi.PieceInfo{}, xerrors.Errorf("rename temp piece to dest %s -> %s: %w", tempDest, dest, err) + } + + removeTemp = false + return abi.PieceInfo{PieceCID: pcid, Size: psize}, nil +} diff --git a/lib/testutils/testutils.go b/lib/testutils/testutils.go index 9ca88c8a6..6598dcad4 100644 --- a/lib/testutils/testutils.go +++ b/lib/testutils/testutils.go @@ -7,8 +7,8 @@ import ( "io" "math/bits" "os" - "path" "strings" + "time" "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" @@ -26,12 +26,14 @@ import ( carv2 "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" "github.com/multiformats/go-multihash" + "github.com/oklog/ulid" "golang.org/x/xerrors" "github.com/filecoin-project/go-data-segment/datasegment" - commcid "github.com/filecoin-project/go-fil-commcid" commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/commcidv2" ) const defaultHashFunction = uint64(multihash.BLAKE2B_MIN + 31) @@ -172,7 +174,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, return nd.Cid(), nil } -func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggregateOut bool) (cid.Cid, abi.PaddedPieceSize, error) { +func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggregateOut bool) (cid.Cid, error) { var lines []string var readers []io.Reader var deals []abi.PieceInfo @@ -180,98 +182,107 @@ func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggre for _, f := range files { file, err := os.Open(f) if err != nil { - return cid.Undef, 0, xerrors.Errorf("opening subpiece file: %w", err) + return cid.Undef, xerrors.Errorf("opening subpiece file: %w", err) } stat, err := file.Stat() if err != nil { - return cid.Undef, 0, xerrors.Errorf("getting file stat: %w", err) + return cid.Undef, xerrors.Errorf("getting file stat: %w", err) } cp := new(commp.Calc) _, err = io.Copy(cp, file) if err != nil { - return cid.Undef, 0, xerrors.Errorf("copying subpiece to commp writer: %w", err) + return cid.Undef, xerrors.Errorf("copying subpiece to commp writer: %w", err) } _, err = file.Seek(0, io.SeekStart) if err != nil { - return cid.Undef, 0, xerrors.Errorf("seeking to start of file: %w", err) + return cid.Undef, xerrors.Errorf("seeking to start of file: %w", err) } pbytes, size, err := cp.Digest() if err != nil { - return cid.Undef, 0, xerrors.Errorf("computing digest for subpiece: %w", err) + return cid.Undef, xerrors.Errorf("computing digest for subpiece: %w", err) } - pcid, err := commcid.DataCommitmentV1ToCID(pbytes) + comm, err := commcidv2.NewSha2CommP(uint64(stat.Size()), pbytes) if err != nil { - return cid.Undef, 0, xerrors.Errorf("converting data commitment to CID: %w", err) + return cid.Undef, xerrors.Errorf("converting data commitment to CID: %w", err) } deals = append(deals, abi.PieceInfo{ - PieceCID: pcid, + PieceCID: comm.PCidV1(), Size: abi.PaddedPieceSize(size), }) readers = append(readers, file) urlStr := fmt.Sprintf("http://piece-server:12320/pieces?id=%s", stat.Name()) - lines = append(lines, fmt.Sprintf("%s\t%d\t%d\t%s", pcid.String(), size, stat.Size(), urlStr)) + lines = append(lines, fmt.Sprintf("%s\t%s", comm.PCidV2().String(), urlStr)) } _, upsize, err := datasegment.ComputeDealPlacement(deals) if err != nil { - return cid.Undef, 0, xerrors.Errorf("computing deal placement: %w", err) + return cid.Undef, xerrors.Errorf("computing deal placement: %w", err) } next := 1 << (64 - bits.LeadingZeros64(upsize+256)) if abi.PaddedPieceSize(next) != dealSize { - return cid.Undef, 0, fmt.Errorf("deal size mismatch: expected %d, got %d", dealSize, abi.PaddedPieceSize(next)) + return cid.Undef, fmt.Errorf("deal size mismatch: expected %d, got %d", dealSize, abi.PaddedPieceSize(next)) } a, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), deals) if err != nil { - return cid.Undef, 0, xerrors.Errorf("creating aggregate: %w", err) + return cid.Undef, xerrors.Errorf("creating aggregate: %w", err) } out, err := a.AggregateObjectReader(readers) if err != nil { - return cid.Undef, 0, xerrors.Errorf("creating aggregate reader: %w", err) + return cid.Undef, xerrors.Errorf("creating aggregate reader: %w", err) } - p := path.Dir(files[0]) + x, err := ulid.New(uint64(time.Now().UnixMilli()), rand.Reader) + if err != nil { + return cid.Undef, xerrors.Errorf("creating aggregate file: %w", err) + } - f, err := os.CreateTemp(p, "aggregate_*") + f, err := os.OpenFile(x.String(), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644) if err != nil { - return cid.Undef, 0, err + return cid.Undef, err } defer f.Close() cp := new(commp.Calc) w := io.MultiWriter(cp, f) - _, err = io.Copy(w, out) + n, err := io.Copy(w, out) if err != nil { - return cid.Undef, 0, xerrors.Errorf("writing aggregate: %w", err) + f.Close() + return cid.Undef, xerrors.Errorf("writing aggregate: %w", err) } + f.Close() + digest, paddedPieceSize, err := cp.Digest() if err != nil { - return cid.Undef, 0, xerrors.Errorf("computing digest: %w", err) + return cid.Undef, xerrors.Errorf("computing digest: %w", err) } if abi.PaddedPieceSize(paddedPieceSize) != dealSize { - return cid.Undef, 0, fmt.Errorf("deal size mismatch after final commP: expected %d, got %d", dealSize, abi.PaddedPieceSize(paddedPieceSize)) + return cid.Undef, fmt.Errorf("deal size mismatch after final commP: expected %d, got %d", dealSize, abi.PaddedPieceSize(paddedPieceSize)) + } + + if n != int64(dealSize.Unpadded()) { + return cid.Undef, fmt.Errorf("incorrect aggregate raw size: expected %d, got %d", dealSize.Unpadded(), n) } - pcid, err := commcid.DataCommitmentV1ToCID(digest) + comm, err := commcidv2.NewSha2CommP(uint64(n), digest) if err != nil { - return cid.Undef, 0, xerrors.Errorf("converting digest to CID: %w", err) + return cid.Undef, xerrors.Errorf("creating commP: %w", err) } - err = os.WriteFile(fmt.Sprintf("aggregate_%s", pcid.String()), []byte(strings.Join(lines, "\n")), 0644) + err = os.WriteFile(fmt.Sprintf("aggregate_%s", comm.PCidV2().String()), []byte(strings.Join(lines, "\n")), 0644) if err != nil { - return cid.Undef, 0, xerrors.Errorf("writing aggregate to file: %w", err) + return cid.Undef, xerrors.Errorf("writing aggregate to file: %w", err) } if !aggregateOut { defer os.Remove(f.Name()) } else { - cn := path.Join(p, pcid.String()) - defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", cn)) //nolint:errcheck + defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", comm.PCidV2().String())) //nolint:errcheck } - return pcid, abi.PaddedPieceSize(paddedPieceSize), nil + return comm.PCidV2(), nil } diff --git a/market/mk12/mk12.go b/market/mk12/mk12.go index 3ac6b28d0..89b3aef59 100644 --- a/market/mk12/mk12.go +++ b/market/mk12/mk12.go @@ -540,12 +540,12 @@ func (m *MK12) processDeal(ctx context.Context, deal *ProviderDealState) (*Provi // Store the deal n, err := tx.Exec(`INSERT INTO market_mk12_deals (uuid, signed_proposal_cid, - proposal_signature, proposal, proposal_cid, piece_cid, + proposal_signature, proposal, proposal_cid, piece_cid, raw_size, piece_size, offline, verified, sp_id, start_epoch, end_epoch, client_peer_id, fast_retrieval, announce_to_ipni, url, url_headers, label) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19) ON CONFLICT (uuid) DO NOTHING`, - deal.DealUuid.String(), deal.SignedProposalCID.String(), sigByte, propJson, propCid, prop.PieceCID.String(), + deal.DealUuid.String(), deal.SignedProposalCID.String(), sigByte, propJson, propCid, prop.PieceCID.String(), deal.Transfer.Size, prop.PieceSize, deal.IsOffline, prop.VerifiedDeal, mid, prop.StartEpoch, prop.EndEpoch, deal.ClientPeerID.String(), deal.FastRetrieval, deal.AnnounceToIPNI, tInfo.URL, headers, b.Bytes()) diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index e632c7bc1..ed65af516 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -34,9 +34,6 @@ type DDOV1 struct { // Provider specifies the address of the provider Provider address.Address `json:"provider"` - // Client represents the address of the deal client - Client address.Address `json:"client"` - // Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs PieceManager address.Address `json:"piece_manager"` @@ -61,12 +58,6 @@ type DDOV1 struct { // NotificationPayload holds the notification data typically in a serialized byte array format. NotificationPayload []byte `json:"notification_payload"` - - // Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval - Indexing bool `json:"indexing"` - - // AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). - AnnounceToIPNI bool `json:"announce_to_ipni"` } func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { @@ -92,10 +83,6 @@ func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e return ErrProductValidationFailed, xerrors.Errorf("provider is disabled") } - if d.Client == address.Undef || d.Client.Empty() { - return ErrProductValidationFailed, xerrors.Errorf("client address is not set") - } - if d.PieceManager == address.Undef || d.PieceManager.Empty() { return ErrProductValidationFailed, xerrors.Errorf("piece manager address is not set") } @@ -128,10 +115,6 @@ func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e return ErrProductValidationFailed, xerrors.Errorf("contract verify method is not set") } - if !d.Indexing && d.AnnounceToIPNI { - return ErrProductValidationFailed, xerrors.Errorf("deal cannot be announced to IPNI without indexing") - } - return Ok, nil } @@ -205,3 +188,5 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. func (d *DDOV1) ProductName() ProductName { return ProductNameDDOV1 } + +var _ product = &DDOV1{} diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md index ad8fd87ca..56fb9e2fe 100644 --- a/market/mk20/http/info.md +++ b/market/mk20/http/info.md @@ -109,7 +109,9 @@ Deal represents a structure defining the details and components of a specific de | Field | Type | Tag | Description | |-------|------|-----|-------------| | Identifier | [ulid.ULID](https://pkg.go.dev/github.com/oklog/ulid#ULID) | json:"identifier" | Identifier represents a unique identifier for the deal in UUID format. | -| Data | [mk20.DataSource](#datasource) | json:"data" | Data represents the source of piece data and associated metadata. | +| Client | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"client" | Client wallet for the deal | +| Signature | [[]byte](https://pkg.go.dev/builtin#byte) | json:"signature" | Signature bytes for the client deal | +| Data | [*mk20.DataSource](#datasource) | json:"data" | Data represents the source of piece data and associated metadata. | | Products | [mk20.Products](#products) | json:"products" | Products represents a collection of product-specific information associated with a deal | ### DataSource @@ -118,8 +120,7 @@ DataSource represents the source of piece data, including metadata and optional | Field | Type | Tag | Description | |-------|------|-----|-------------| -| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | PieceCID represents the unique identifier for a piece of data, stored as a CID object. | -| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"piece_size" | Size represents the size of the padded piece in the data source. | +| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. | | Format | [mk20.PieceDataFormat](#piecedataformat) | json:"format" | Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. | | SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"source_http" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | | SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"source_aggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | @@ -131,6 +132,25 @@ DataSource represents the source of piece data, including metadata and optional | Field | Type | Tag | Description | |-------|------|-----|-------------| | DDOV1 | [*mk20.DDOV1](#ddov1) | json:"ddo_v1" | DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) | +| RetrievalV1 | [*mk20.RetrievalV1](#retrievalv1) | json:"retrieval_v1" | RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags. | +| PDPV1 | [*mk20.PDPV1](#pdpv1) | json:"pdp_v1" | PDPV1 represents product-specific configuration for PDP version 1 deals. | + +### DBDDOV1 + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| DDO | [*mk20.DDOV1](#ddov1) | json:"ddo" | | +| DealID | [string](https://pkg.go.dev/builtin#string) | json:"deal_id" | | +| Complete | [bool](https://pkg.go.dev/builtin#bool) | json:"complete" | | +| Error | [sql.NullString](https://pkg.go.dev/database/sql#NullString) | json:"error" | | + +### DBPDPV1 + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| PDP | [*mk20.PDPV1](#pdpv1) | json:"pdp" | | +| Complete | [bool](https://pkg.go.dev/builtin#bool) | json:"complete" | | +| Error | [sql.NullString](https://pkg.go.dev/database/sql#NullString) | json:"error" | | ### DDOV1 @@ -140,7 +160,6 @@ for a DDO deal handling. | Field | Type | Tag | Description | |-------|------|-----|-------------| | Provider | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"provider" | Provider specifies the address of the provider | -| Client | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"client" | Client represents the address of the deal client | | PieceManager | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"piece_manager" | Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs | | Duration | [abi.ChainEpoch](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#ChainEpoch) | json:"duration" | Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400 | | AllocationId | [*verifreg.AllocationId](https://pkg.go.dev/github.com/filecoin-project/go-state-types/builtin/v16/verifreg#AllocationId) | json:"allocation_id" | AllocationId represents an aggregated allocation identifier for the deal. | @@ -149,8 +168,6 @@ for a DDO deal handling. | ContractVerifyMethodParams | [[]byte](https://pkg.go.dev/builtin#byte) | json:"contract_verify_method_params" | ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract | | NotificationAddress | [string](https://pkg.go.dev/builtin#string) | json:"notification_address" | NotificationAddress specifies the address to which notifications will be relayed to when sector is activated | | NotificationPayload | [[]byte](https://pkg.go.dev/builtin#byte) | json:"notification_payload" | NotificationPayload holds the notification data typically in a serialized byte array format. | -| Indexing | [bool](https://pkg.go.dev/builtin#bool) | json:"indexing" | Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval | -| AnnounceToIPNI | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_to_ipni" | AnnounceToIPNI indicates whether the deal should be announced to the Interplanetary Network Indexer (IPNI). | ### DataSourceAggregate @@ -162,11 +179,10 @@ DataSourceAggregate represents an aggregated data source containing multiple ind ### DataSourceHTTP -DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. +DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including associated URLs. | Field | Type | Tag | Description | |-------|------|-----|-------------| -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"rawsize" | RawSize specifies the raw size of the data in bytes. | | URLs | [[]mk20.HttpUrl](#httpurl) | json:"urls" | URLs lists the HTTP endpoints where the piece data can be fetched. | ### DataSourceHttpPut @@ -175,15 +191,13 @@ DataSourceHttpPut represents a data source allowing clients to push piece data a | Field | Type | Tag | Description | |-------|------|-----|-------------| -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | RawSize specifies the raw size of the data in bytes. | ### DataSourceOffline -DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. +DataSourceOffline represents the data source for offline pieces. | Field | Type | Tag | Description | |-------|------|-----|-------------| -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | RawSize specifies the raw size of the data in bytes. | ### DealStatusResponse @@ -201,7 +215,7 @@ FormatAggregate represents the aggregated format for piece data, identified by i | Field | Type | Tag | Description | |-------|------|-----|-------------| | Type | [mk20.AggregateType](https://pkg.go.dev/github.com/filecoin-project/curio/market/mk20#AggregateType) | json:"type" | Type specifies the type of aggregation for data pieces, represented by an AggregateType value. | -| Sub | [[]mk20.PieceDataFormat](#piecedataformat) | json:"sub" | Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate | +| Sub | [[]mk20.DataSource](#datasource) | json:"sub" | Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate | ### FormatBytes @@ -228,6 +242,15 @@ HttpUrl represents an HTTP endpoint configuration for fetching piece data. | Priority | [int](https://pkg.go.dev/builtin#int) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | | Fallback | [bool](https://pkg.go.dev/builtin#bool) | json:"fallback" | Fallback indicates whether this URL serves as a fallback option when other URLs fail. | +### PDPV1 + +PDPV1 represents configuration for product-specific PDP version 1 deals. + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| ProofSetID | [uint64](https://pkg.go.dev/builtin#uint64) | json:"proof_set_id" | | +| DeleteRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_root" | DeleteRoot indicates whether the root of the data should be deleted. This basically means end of deal lifetime. | + ### PieceDataFormat PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. @@ -238,6 +261,24 @@ PieceDataFormat represents various formats in which piece data can be defined, i | Aggregate | [*mk20.FormatAggregate](#formataggregate) | json:"aggregate" | Aggregate holds a reference to the aggregated format of piece data. | | Raw | [*mk20.FormatBytes](#formatbytes) | json:"raw" | Raw represents the raw format of the piece data, encapsulated as bytes. | +### PieceInfo + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| PieceCIDV1 | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | | +| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"size" | | +| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | | + +### RetrievalV1 + +RetrievalV1 defines a structure for managing retrieval settings + +| Field | Type | Tag | Description | +|-------|------|-----|-------------| +| Indexing | [bool](https://pkg.go.dev/builtin#bool) | json:"indexing" | Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval | +| AnnouncePayload | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_payload" | AnnouncePayload indicates whether the payload should be announced to IPNI. | +| AnnouncePiece | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_piece" | AnnouncePiece indicates whether the piece information should be announced to IPNI. | + ### StartUpload StartUpload represents metadata for initiating an upload operation, containing the chunk size of the data to be uploaded. diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index d0e37a170..d84a7b120 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/multictladdr" "github.com/filecoin-project/curio/lib/paths" @@ -45,11 +46,11 @@ type MK20 struct { cfg *config.CurioConfig sm map[address.Address]abi.SectorSize as *multictladdr.MultiAddressSelector - stor paths.StashStore + sc *ffi.SealCalls maxParallelUploads *atomic.Int64 } -func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, stor paths.StashStore) (*MK20, error) { +func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, sc *ffi.SealCalls) (*MK20, error) { ctx := context.Background() // Ensure MinChunk size and max chunkSize is a power of 2 @@ -73,7 +74,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } } - ret := &MK20{ + return &MK20{ miners: miners, db: db, api: mapi, @@ -82,12 +83,9 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI cfg: cfg, sm: sm, as: as, - stor: stor, + sc: sc, maxParallelUploads: new(atomic.Int64), - } - - go ret.MarkChunkComplete(ctx) - return ret, nil + }, nil } func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { @@ -157,7 +155,9 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje if err != nil { return false, err } - n, err := tx.Exec(`Update market_mk20_deal SET market_deal_id = $1 WHERE id = $2`, id, deal.Identifier.String()) + n, err := tx.Exec(`UPDATE market_mk20_deal + SET ddo_v1 = jsonb_set(ddo_v1, '{deal_id}', to_jsonb($1::text)) + WHERE id = $2;`, id, deal.Identifier.String()) if err != nil { return false, err } @@ -206,7 +206,16 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - if deal.Data.Size > abi.PaddedPieceSize(m.sm[deal.Products.DDOV1.Provider]) { + size, err := deal.Size() + if err != nil { + log.Errorw("error getting deal size", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Error getting deal size from PieceCID", + }, nil + } + + if size > abi.PaddedPieceSize(m.sm[deal.Products.DDOV1.Provider]) { return &ProviderDealRejectionInfo{ HTTPCode: http.StatusBadRequest, Reason: "Deal size is larger than the miner's sector size", @@ -214,23 +223,25 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe } if deal.Data.Format.Raw != nil { - if deal.Products.DDOV1.Indexing { - return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, - Reason: "Raw bytes deal cannot be indexed", - }, nil + if deal.Products.RetrievalV1 != nil { + if deal.Products.RetrievalV1.Indexing { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Raw bytes deal cannot be indexed", + }, nil + } } } if deal.Products.DDOV1.AllocationId != nil { - if deal.Data.Size < abi.PaddedPieceSize(verifreg.MinimumVerifiedAllocationSize) { + if size < abi.PaddedPieceSize(verifreg.MinimumVerifiedAllocationSize) { return &ProviderDealRejectionInfo{ HTTPCode: http.StatusBadRequest, Reason: "Verified piece size must be at least 1MB", }, nil } - alloc, err := m.api.StateGetAllocation(ctx, deal.Products.DDOV1.Client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) + alloc, err := m.api.StateGetAllocation(ctx, deal.Client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) if err != nil { return &ProviderDealRejectionInfo{ HTTPCode: http.StatusInternalServerError, @@ -244,7 +255,7 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - clientID, err := address.IDFromAddress(deal.Products.DDOV1.Client) + clientID, err := address.IDFromAddress(deal.Client) if err != nil { return &ProviderDealRejectionInfo{ HTTPCode: http.StatusBadRequest, @@ -280,7 +291,7 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - if deal.Data.Size != alloc.Size { + if size != alloc.Size { return &ProviderDealRejectionInfo{ HTTPCode: http.StatusBadRequest, Reason: "Allocation size does not match the piece size", diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index 248e21998..dad677151 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -9,19 +9,17 @@ import ( "io" "math" "net/http" - "os" - "strings" "time" - commcid "github.com/filecoin-project/go-fil-commcid" - commp "github.com/filecoin-project/go-fil-commp-hashhash" - "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/lib/dealdata" + "github.com/filecoin-project/curio/lib/storiface" ) func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.ResponseWriter) { @@ -46,9 +44,9 @@ func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.Resp err = m.db.QueryRow(ctx, `SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE complete) AS complete, - COUNT(*) FILTER (WHERE ref_id IS NULL) AS missing, + COUNT(*) FILTER (WHERE NOT complete) AS missing, ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE complete) AS completed_chunks, - ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE ref_id IS NULL) AS incomplete_chunks + ARRAY_AGG(chunk ORDER BY chunk) FILTER (WHERE NOT complete) AS incomplete_chunks FROM market_mk20_deal_chunk WHERE @@ -150,7 +148,7 @@ func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, chunkSize in return } - rawSize, err := deal.Data.RawSize() + rawSize, err := deal.RawSize() if err != nil { log.Errorw("failed to get raw size of deal", "deal", id, "error", err) http.Error(w, "", http.StatusInternalServerError) @@ -257,100 +255,37 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w chunkSize := chunkDetails[0].Size reader := NewTimeoutReader(data, time.Second*5) m.maxParallelUploads.Add(1) - wr := new(commp.Calc) - - failed := true - - // Function to write data into StashStore and calculate commP - writeFunc := func(f *os.File) error { - limitedReader := io.LimitReader(reader, chunkSize+1) // +1 to detect exceeding the limit - writer := io.MultiWriter(f, wr) - - size, err := io.CopyBuffer(writer, limitedReader, make([]byte, 4<<20)) - if err != nil { - return fmt.Errorf("failed to read and write chunk data: %w", err) - } - - if size > chunkSize { - return fmt.Errorf("chunk data exceeds the maximum allowed size") - } - - if chunkSize != size { - return fmt.Errorf("chunk size %d does not match with uploaded data size %d", chunkSize, size) - } - - return nil - } - - // Upload into StashStore - stashID, err := m.stor.StashCreate(ctx, chunkSize, writeFunc) - if err != nil { - if err.Error() == "chunk data exceeds the maximum allowed size" { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, "chunk data exceeds the maximum allowed size", http.StatusRequestEntityTooLarge) - return - } else if strings.Contains(err.Error(), "does not match with uploaded data") { - log.Errorw("Storing", "Deal", id, "error", err) - http.Error(w, errors.Unwrap(err).Error(), http.StatusBadRequest) - return - } else { - log.Errorw("Failed to store piece data in StashStore", "error", err) - http.Error(w, "Failed to store piece data", http.StatusInternalServerError) - return - } - } - defer func() { - if failed { - err = m.stor.StashRemove(ctx, stashID) - if err != nil { - log.Errorw("Failed to remove stash file", "Deal", id, "error", err) - } - } - }() - - digest, pieceSize, err := wr.Digest() - if err != nil { - log.Errorw("failed to calculate commP", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - pcid, err := commcid.DataCommitmentV1ToCID(digest) + // Generate unique tmp pieceCID and Size for parked_pieces tables + wr := new(commp.Calc) + n, err := wr.Write([]byte(fmt.Sprintf("%s, %d, %d, %s", id.String(), chunk, chunkSize, time.Now().String()))) if err != nil { - log.Errorw("failed to calculate piece CID", "deal", id, "chunk", chunk, "error", err) + log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "chunk", chunk, "error", err) http.Error(w, "", http.StatusInternalServerError) return } - - pSize := abi.PaddedPieceSize(pieceSize) - - log.Debugw("uploaded chunk to stash store", "deal", id, "chunk", chunk, "stashID", stashID.String()) - - stashUrl, err := m.stor.StashURL(stashID) + digest, tsize, err := wr.Digest() if err != nil { - log.Errorw("Failed to get stash url", "error", err) - http.Error(w, "", http.StatusInternalServerError) - return + panic(err) } - stashUrl.Scheme = dealdata.CustoreScheme + tpcid := cid.NewCidV1(cid.FilCommitmentUnsealed, digest) + var pnum, refID int64 - log.Debugw("uploading chunk generated URL", "deal", id, "chunk", chunk, "url", stashUrl.String()) + // Generate piece park details with tmp pieceCID and Size comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - var pieceID int64 err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 - AND piece_raw_size = $3`, pcid.String(), pSize, chunkSize).Scan(&pieceID) + AND piece_raw_size = $3`, tpcid.String(), tsize, n).Scan(&pnum) if err != nil { if errors.Is(err, pgx.ErrNoRows) { err = tx.QueryRow(` - INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - VALUES ($1, $2, $3, FALSE) + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) + VALUES ($1, $2, $3, FALSE, TRUE) ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING - RETURNING id`, pcid.String(), int64(pieceSize), chunkSize).Scan(&pieceID) + RETURNING id`, tpcid.String(), tsize, n).Scan(&pnum) if err != nil { return false, xerrors.Errorf("inserting new parked piece and getting id: %w", err) } @@ -360,14 +295,68 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } // Add parked_piece_ref - var refID int64 err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, FALSE) RETURNING ref_id`, pieceID, stashUrl.String()).Scan(&refID) + VALUES ($1, $2, FALSE) RETURNING ref_id`, pnum, "/PUT").Scan(&refID) if err != nil { return false, xerrors.Errorf("inserting parked piece ref: %w", err) } - n, err := tx.Exec(`UPDATE market_mk20_deal_chunk SET ref_id = $1 + return true, nil + }) + + if err != nil { + log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if !comm { + log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", "failed to commit transaction") + http.Error(w, "", http.StatusInternalServerError) + return + } + + log.Debugw("tmp piece details generated for the chunk", "deal", id, "chunk", chunk) + + failed := true + defer func() { + if failed { + _, err = m.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + log.Errorw("failed to delete parked piece ref", "deal", id, "chunk", chunk, "error", err) + } + } + }() + + // Store the piece and generate PieceCID and Size + pi, err := m.sc.WriteUploadPiece(ctx, storiface.PieceNumber(pnum), chunkSize, reader, storiface.PathSealing) + if err != nil { + log.Errorw("failed to write piece", "deal", id, "chunk", chunk, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + log.Debugw("piece stored", "deal", id, "chunk", chunk) + + // Update piece park details with correct values + comm, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE parked_pieces SET + piece_cid = $1, + piece_padded_size = $2, + piece_raw_size = $3, + complete = true + WHERE id = $4`, + pi.PieceCID.String(), pi.Size, chunkSize, pnum) + if err != nil { + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + + n, err = tx.Exec(`UPDATE market_mk20_deal_chunk SET + complete = TRUE, + ref_id = $1 WHERE id = $2 AND chunk = $3 AND complete = FALSE @@ -391,6 +380,8 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w return } + log.Debugw("chunk upload finished", "deal", id, "chunk", chunk) + failed = false } @@ -430,63 +421,63 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { w.WriteHeader(http.StatusOK) } -func (m *MK20) MarkChunkComplete(ctx context.Context) { - ticker := time.NewTicker(time.Second * 3) - defer ticker.Stop() - for { - select { - case <-ticker.C: - markChunksComplete(ctx, m.db) - case <-ctx.Done(): - return - } - } -} - -func markChunksComplete(ctx context.Context, db *harmonydb.DB) { - var chunks []struct { - ID string `db:"id"` - Chunk int `db:"chunk"` - ChunkSize int64 `db:"chunk_size"` - Complete bool `db:"complete"` - RefId int64 `db:"ref_id"` - } - - err := db.Select(ctx, &chunks, `SELECT id, - chunk, - chunk_size, - ref_id, - complete - FROM market_mk20_deal_chunk - WHERE finalize = FALSE - AND complete = FALSE - AND ref_id IS NOT NULL`) - if err != nil { - log.Errorw("failed to get chunks to mark complete", "error", err) - return - } - for _, chunk := range chunks { - var complete bool - err := db.QueryRow(ctx, `SELECT p.complete - FROM parked_pieces AS p - JOIN parked_piece_refs AS r - ON r.piece_id = p.id - WHERE r.ref_id = $1`, chunk.RefId).Scan(&complete) - if err != nil { - log.Errorw("failed to get piece complete status", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) - continue - } - if complete { - _, err := db.Exec(ctx, `UPDATE market_mk20_deal_chunk - SET complete = TRUE - WHERE id = $1 - AND chunk = $2 - AND ref_id = $3 - AND finalize = FALSE`, chunk.ID, chunk.Chunk, chunk.RefId) - if err != nil { - log.Errorw("failed to mark chunk complete", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) - continue - } - } - } -} +//func (m *MK20) MarkChunkComplete(ctx context.Context) { +// ticker := time.NewTicker(time.Second * 3) +// defer ticker.Stop() +// for { +// select { +// case <-ticker.C: +// markChunksComplete(ctx, m.db) +// case <-ctx.Done(): +// return +// } +// } +//} +// +//func markChunksComplete(ctx context.Context, db *harmonydb.DB) { +// var chunks []struct { +// ID string `db:"id"` +// Chunk int `db:"chunk"` +// ChunkSize int64 `db:"chunk_size"` +// Complete bool `db:"complete"` +// RefId int64 `db:"ref_id"` +// } +// +// err := db.Select(ctx, &chunks, `SELECT id, +// chunk, +// chunk_size, +// ref_id, +// complete +// FROM market_mk20_deal_chunk +// WHERE finalize = FALSE +// AND complete = FALSE +// AND ref_id IS NOT NULL`) +// if err != nil { +// log.Errorw("failed to get chunks to mark complete", "error", err) +// return +// } +// for _, chunk := range chunks { +// var complete bool +// err := db.QueryRow(ctx, `SELECT p.complete +// FROM parked_pieces AS p +// JOIN parked_piece_refs AS r +// ON r.piece_id = p.id +// WHERE r.ref_id = $1`, chunk.RefId).Scan(&complete) +// if err != nil { +// log.Errorw("failed to get piece complete status", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) +// continue +// } +// if complete { +// _, err := db.Exec(ctx, `UPDATE market_mk20_deal_chunk +// SET complete = TRUE +// WHERE id = $1 +// AND chunk = $2 +// AND ref_id = $3 +// AND finalize = FALSE`, chunk.ID, chunk.Chunk, chunk.RefId) +// if err != nil { +// log.Errorw("failed to mark chunk complete", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) +// continue +// } +// } +// } +//} diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go new file mode 100644 index 000000000..f0d0483e6 --- /dev/null +++ b/market/mk20/pdp_v1.go @@ -0,0 +1,28 @@ +package mk20 + +import ( + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +// PDPV1 represents configuration for product-specific PDP version 1 deals. +type PDPV1 struct { + ProofSetID uint64 `json:"proof_set_id"` + + // DeleteRoot indicates whether the root of the data should be deleted. This basically means end of deal lifetime. + DeleteRoot bool `json:"delete_root"` +} + +func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { + code, err := IsProductEnabled(db, p.ProductName()) + if err != nil { + return code, err + } + return Ok, nil +} + +func (p *PDPV1) ProductName() ProductName { + return ProductNamePDPV1 +} + +var _ product = &PDPV1{} diff --git a/market/mk20/retrieval_v1.go b/market/mk20/retrieval_v1.go new file mode 100644 index 000000000..365771b58 --- /dev/null +++ b/market/mk20/retrieval_v1.go @@ -0,0 +1,38 @@ +package mk20 + +import ( + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" +) + +// RetrievalV1 defines a structure for managing retrieval settings +type RetrievalV1 struct { + // Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval + Indexing bool `json:"indexing"` + + // AnnouncePayload indicates whether the payload should be announced to IPNI. + AnnouncePayload bool `json:"announce_payload"` + + // AnnouncePiece indicates whether the piece information should be announced to IPNI. + AnnouncePiece bool `json:"announce_piece"` +} + +func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { + code, err := IsProductEnabled(db, r.ProductName()) + if err != nil { + return code, err + } + + if !r.Indexing && r.AnnouncePayload { + return ErrProductValidationFailed, xerrors.Errorf("deal cannot be announced to IPNI without indexing") + } + return Ok, nil +} + +func (r *RetrievalV1) ProductName() ProductName { + return ProductNameRetrievalV1 +} + +var _ product = &RetrievalV1{} diff --git a/market/mk20/types.go b/market/mk20/types.go index 0e7f8a03b..4f0e2149c 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -6,7 +6,10 @@ import ( "github.com/ipfs/go-cid" "github.com/oklog/ulid" - "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" ) // Deal represents a structure defining the details and components of a specific deal in the system. @@ -15,8 +18,14 @@ type Deal struct { // Identifier represents a unique identifier for the deal in UUID format. Identifier ulid.ULID `json:"identifier"` + // Client wallet for the deal + Client address.Address `json:"client"` + + // Signature bytes for the client deal + Signature []byte `json:"signature"` + // Data represents the source of piece data and associated metadata. - Data DataSource `json:"data"` + Data *DataSource `json:"data"` // Products represents a collection of product-specific information associated with a deal Products Products `json:"products"` @@ -25,17 +34,20 @@ type Deal struct { type Products struct { // DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) DDOV1 *DDOV1 `json:"ddo_v1"` + + // RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags. + RetrievalV1 *RetrievalV1 `json:"retrieval_v1"` + + // PDPV1 represents product-specific configuration for PDP version 1 deals. + PDPV1 *PDPV1 `json:"pdp_v1"` } // DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. type DataSource struct { - // PieceCID represents the unique identifier for a piece of data, stored as a CID object. + // PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. PieceCID cid.Cid `json:"piece_cid"` - // Size represents the size of the padded piece in the data source. - Size abi.PaddedPieceSize `json:"piece_size"` - // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` @@ -76,31 +88,24 @@ type FormatAggregate struct { // Type specifies the type of aggregation for data pieces, represented by an AggregateType value. Type AggregateType `json:"type"` - // Sub holds a slice of PieceDataFormat, representing various formats of piece data aggregated under this format. + // Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format. // The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate - Sub []PieceDataFormat `json:"sub"` + Sub []DataSource `json:"sub"` } // FormatBytes defines the raw byte representation of data as a format. type FormatBytes struct{} -// DataSourceOffline represents the data source for offline pieces, including metadata such as the raw size of the piece. -type DataSourceOffline struct { - // RawSize specifies the raw size of the data in bytes. - RawSize uint64 `json:"raw_size"` -} +// DataSourceOffline represents the data source for offline pieces. +type DataSourceOffline struct{} // DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. type DataSourceAggregate struct { Pieces []DataSource `json:"pieces"` } -// DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including its raw size and associated URLs. +// DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including associated URLs. type DataSourceHTTP struct { - - // RawSize specifies the raw size of the data in bytes. - RawSize uint64 `json:"rawsize"` - // URLs lists the HTTP endpoints where the piece data can be fetched. URLs []HttpUrl `json:"urls"` } @@ -122,10 +127,7 @@ type HttpUrl struct { } // DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted. -type DataSourceHttpPut struct { - // RawSize specifies the raw size of the data in bytes. - RawSize uint64 `json:"raw_size"` -} +type DataSourceHttpPut struct{} // AggregateType represents an unsigned integer used to define the type of aggregation for data pieces in the system. type AggregateType int @@ -186,7 +188,9 @@ type ProductName string const ( // ProductNameDDOV1 represents the identifier for the "ddo_v1" product used in contract operations and validations. - ProductNameDDOV1 ProductName = "ddo_v1" + ProductNameDDOV1 ProductName = "ddo_v1" + ProductNamePDPV1 ProductName = "pdp_v1" + ProductNameRetrievalV1 ProductName = "retrieval_v1" ) type DataSourceName string @@ -199,3 +203,8 @@ const ( DataSourceNamePDP DataSourceName = "pdp" DataSourceNamePut DataSourceName = "put" ) + +type product interface { + Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) + ProductName() ProductName +} diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 31581344b..8c9ee7316 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "math/bits" "net/http" "net/url" "time" @@ -18,16 +17,28 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-data-segment/datasegment" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/commcidv2" + + "github.com/filecoin-project/lotus/lib/sigs" ) func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { - code, err := d.Products.Validate(db, cfg) + if d.Client.Empty() { + return ErrBadProposal, xerrors.Errorf("no client") + } + + code, err := d.ValidateSignature() + if err != nil { + return code, xerrors.Errorf("signature validation failed: %w", err) + } + + code, err = d.Products.Validate(db, cfg) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) } @@ -35,14 +46,39 @@ func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, er return d.Data.Validate(db) } -func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { +func (d *Deal) ValidateSignature() (ErrorCode, error) { + if len(d.Signature) == 0 { + return ErrBadProposal, xerrors.Errorf("no signature") + } - if !d.PieceCID.Defined() { - return ErrBadProposal, xerrors.Errorf("piece cid is not defined") + sig := &crypto.Signature{} + err := sig.UnmarshalBinary(d.Signature) + if err != nil { + return ErrBadProposal, xerrors.Errorf("invalid signature") } - if d.Size == 0 { - return ErrBadProposal, xerrors.Errorf("piece size is 0") + msg, err := d.Identifier.MarshalBinary() + if err != nil { + return ErrBadProposal, xerrors.Errorf("invalid identifier") + } + + if sig.Type == crypto.SigTypeBLS || sig.Type == crypto.SigTypeSecp256k1 || sig.Type == crypto.SigTypeDelegated { + err = sigs.Verify(sig, d.Client, msg) + if err != nil { + return ErrBadProposal, xerrors.Errorf("invalid signature") + } + return Ok, nil + } + + // Add more types if required in Future + return ErrBadProposal, xerrors.Errorf("invalid signature type") +} + +func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { + + err := ValidatePieceCID(d.PieceCID) + if err != nil { + return ErrBadProposal, err } if d.SourceOffline != nil && d.SourceHTTP != nil && d.SourceAggregate != nil && d.SourceHttpPut != nil { @@ -76,13 +112,14 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { return ErrMalformedDataSource, xerrors.Errorf("no pieces in aggregate") } - for _, p := range d.SourceAggregate.Pieces { - if !p.PieceCID.Defined() { - return ErrMalformedDataSource, xerrors.Errorf("piece cid is not defined") - } + if len(d.SourceAggregate.Pieces) == 1 { + return ErrMalformedDataSource, xerrors.Errorf("aggregate must have at least 2 pieces") + } - if p.Size == 0 { - return ErrMalformedDataSource, xerrors.Errorf("piece size is 0") + for _, p := range d.SourceAggregate.Pieces { + err := ValidatePieceCID(p.PieceCID) + if err != nil { + return ErrMalformedDataSource, xerrors.Errorf("invalid piece cid") } var ifcar, ifraw bool @@ -120,10 +157,6 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { } if p.SourceHTTP != nil { - if p.SourceHTTP.RawSize == 0 { - return ErrMalformedDataSource, xerrors.Errorf("raw size is 0 for sub piece in aggregate") - } - if len(p.SourceHTTP.URLs) == 0 { return ErrMalformedDataSource, xerrors.Errorf("no urls defined for sub piece in aggregate") } @@ -135,13 +168,6 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { } } } - - if p.SourceOffline != nil { - if p.SourceOffline.RawSize == 0 { - return ErrMalformedDataSource, xerrors.Errorf("raw size is 0 for sub piece in aggregate") - } - } - } } else { if len(d.Format.Aggregate.Sub) == 0 { @@ -168,10 +194,6 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { return code, err } - if d.SourceHTTP.RawSize == 0 { - return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") - } - if len(d.SourceHTTP.URLs) == 0 { return ErrMalformedDataSource, xerrors.Errorf("no urls defined") } @@ -189,10 +211,6 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { if err != nil { return code, err } - - if d.SourceOffline.RawSize == 0 { - return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") - } } if d.SourceHttpPut != nil { @@ -200,181 +218,191 @@ func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { if err != nil { return code, err } - if d.SourceHttpPut.RawSize == 0 { - return ErrMalformedDataSource, xerrors.Errorf("raw size is 0") - } } - raw, err := d.RawSize() - if err != nil { - return ErrBadProposal, err - } + return Ok, nil +} - if padreader.PaddedSize(raw).Padded() != d.Size { - return ErrBadProposal, xerrors.Errorf("invalid size") +func ValidatePieceCID(c cid.Cid) error { + if !c.Defined() { + return xerrors.Errorf("piece cid is not defined") } - return Ok, nil -} + if c.Prefix().Codec != cid.Raw { + return xerrors.Errorf("piece cid is not raw") + } -func (d DataSource) RawSize() (uint64, error) { - if d.Format.Aggregate != nil { - if d.Format.Aggregate.Type == AggregateTypeV1 { - if d.SourceAggregate != nil { - var pinfos []abi.PieceInfo - for _, piece := range d.SourceAggregate.Pieces { - pinfos = append(pinfos, abi.PieceInfo{ - PieceCID: piece.PieceCID, - Size: piece.Size, - }) - } - _, asize, err := datasegment.ComputeDealPlacement(pinfos) - if err != nil { - return 0, err - } - next := 1 << (64 - bits.LeadingZeros64(asize+256)) - if abi.PaddedPieceSize(next) != d.Size { - return 0, xerrors.Errorf("invalid aggregate size") - } + commp, err := commcidv2.CommPFromPCidV2(c) + if err != nil { + return xerrors.Errorf("invalid piece cid: %w", err) + } - a, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), pinfos) - if err != nil { - return 0, err - } + if commp.PieceInfo().Size == 0 { + return xerrors.Errorf("piece size is 0") + } - return uint64(a.DealSize.Unpadded()), nil - } - } + if commp.PayloadSize() == 0 { + return xerrors.Errorf("payload size is 0") } - if d.SourceHTTP != nil { - return d.SourceHTTP.RawSize, nil + if padreader.PaddedSize(commp.PayloadSize()).Padded() != commp.PieceInfo().Size { + return xerrors.Errorf("invalid piece size") } - if d.SourceOffline != nil { - return d.SourceOffline.RawSize, nil + return nil +} + +type PieceInfo struct { + PieceCIDV1 cid.Cid `json:"piece_cid"` + Size abi.PaddedPieceSize `json:"size"` + RawSize uint64 `json:"raw_size"` +} + +func (d *Deal) RawSize() (uint64, error) { + commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) + if err != nil { + return 0, xerrors.Errorf("invalid piece cid: %w", err) } + return commp.PayloadSize(), nil +} - if d.SourceHttpPut != nil { - return d.SourceHttpPut.RawSize, nil +func (d *Deal) Size() (abi.PaddedPieceSize, error) { + commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) + if err != nil { + return 0, xerrors.Errorf("invalid piece cid: %w", err) } + return commp.PieceInfo().Size, nil +} - return 0, xerrors.Errorf("no source defined") +func (d *Deal) PieceInfo() (*PieceInfo, error) { + return GetPieceInfo(d.Data.PieceCID) +} + +func GetPieceInfo(c cid.Cid) (*PieceInfo, error) { + commp, err := commcidv2.CommPFromPCidV2(c) + if err != nil { + return nil, xerrors.Errorf("invalid piece cid: %w", err) + } + return &PieceInfo{ + PieceCIDV1: commp.PCidV1(), + Size: commp.PieceInfo().Size, + RawSize: commp.PayloadSize(), + }, nil } func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { - if d.DDOV1 == nil { - return ErrBadProposal, xerrors.Errorf("no products") + if d.DDOV1 != nil { + code, err := d.DDOV1.Validate(db, cfg) + if err != nil { + return code, err + } + } + if d.RetrievalV1 != nil { + code, err := d.RetrievalV1.Validate(db, cfg) + if err != nil { + return code, err + } } + if d.PDPV1 != nil { + code, err := d.PDPV1.Validate(db, cfg) + if err != nil { + return code, err + } + } + return Ok, nil +} - return d.DDOV1.Validate(db, cfg) +type DBDDOV1 struct { + DDO *DDOV1 `json:"ddo"` + DealID string `json:"deal_id"` + Complete bool `json:"complete"` + Error sql.NullString `json:"error"` +} + +type DBPDPV1 struct { + PDP *PDPV1 `json:"pdp"` + Complete bool `json:"complete"` + Error sql.NullString `json:"error"` } type DBDeal struct { - Identifier string `db:"id"` - SpID int64 `db:"sp_id"` - PieceCID string `db:"piece_cid"` - Size int64 `db:"piece_size"` - Format json.RawMessage `db:"format"` - SourceHTTP json.RawMessage `db:"source_http"` - SourceAggregate json.RawMessage `db:"source_aggregate"` - SourceOffline json.RawMessage `db:"source_offline"` - SourceHttpPut json.RawMessage `db:"source_http_put"` - DDOv1 json.RawMessage `db:"ddo_v1"` - Error sql.NullString `db:"error"` + Identifier string `db:"id"` + Client string `db:"client"` + PieceCIDV2 sql.NullString `db:"piece_cid_v2"` + PieceCID sql.NullString `db:"piece_cid"` + Size sql.NullInt64 `db:"piece_size"` + RawSize sql.NullInt64 `db:"raw_size"` + Data json.RawMessage `db:"data"` + DDOv1 json.RawMessage `db:"ddo_v1"` + RetrievalV1 json.RawMessage `db:"retrieval_v1"` + PDPV1 json.RawMessage `db:"pdp_v1"` } func (d *Deal) ToDBDeal() (*DBDeal, error) { - var err error - // Marshal SourceHTTP (optional) - var sourceHTTPBytes []byte - if d.Data.SourceHTTP != nil { - sourceHTTPBytes, err = json.Marshal(d.Data.SourceHTTP) - if err != nil { - return nil, fmt.Errorf("marshal source_http: %w", err) - } - } else { - sourceHTTPBytes = []byte("null") + ddeal := DBDeal{ + Identifier: d.Identifier.String(), + Client: d.Client.String(), } - // Marshal SourceAggregate (optional) - var sourceAggregateBytes []byte - if d.Data.SourceAggregate != nil { - sourceAggregateBytes, err = json.Marshal(d.Data.SourceAggregate) + if d.Data != nil { + dataBytes, err := json.Marshal(d.Data) if err != nil { - return nil, fmt.Errorf("marshal source_aggregate: %w", err) + return nil, fmt.Errorf("marshal data: %w", err) } - if len(d.Data.SourceAggregate.Pieces) > 0 && len(d.Data.SourceAggregate.Pieces) != len(d.Data.Format.Aggregate.Sub) { - var subPieces []PieceDataFormat - for _, p := range d.Data.SourceAggregate.Pieces { - subPieces = append(subPieces, PieceDataFormat{ - Car: p.Format.Car, - Raw: p.Format.Raw, - Aggregate: p.Format.Aggregate, - }) - } - d.Data.Format.Aggregate.Sub = subPieces + commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) + if err != nil { + return nil, fmt.Errorf("invalid piece cid: %w", err) } + ddeal.PieceCIDV2.String = d.Data.PieceCID.String() + ddeal.PieceCIDV2.Valid = true + ddeal.PieceCID.String = commp.PCidV1().String() + ddeal.PieceCID.Valid = true + ddeal.Size.Int64 = int64(commp.PieceInfo().Size) + ddeal.Size.Valid = true + ddeal.RawSize.Int64 = int64(commp.PayloadSize()) + ddeal.RawSize.Valid = true + ddeal.Data = dataBytes } else { - sourceAggregateBytes = []byte("null") + ddeal.Data = []byte("null") } - // Marshal SourceOffline (optional) - var sourceOfflineBytes []byte - if d.Data.SourceOffline != nil { - sourceOfflineBytes, err = json.Marshal(d.Data.SourceOffline) + if d.Products.DDOV1 != nil { + dddov1 := DBDDOV1{ + DDO: d.Products.DDOV1, + } + ddov1, err := json.Marshal(dddov1) if err != nil { - return nil, fmt.Errorf("marshal source_offline: %w", err) + return nil, fmt.Errorf("marshal ddov1: %w", err) } + ddeal.DDOv1 = ddov1 } else { - sourceOfflineBytes = []byte("null") + ddeal.DDOv1 = []byte("null") } - var sourceHttpPutBytes []byte - if d.Data.SourceHttpPut != nil { - sourceHttpPutBytes, err = json.Marshal(d.Data.SourceHttpPut) + if d.Products.RetrievalV1 != nil { + rev, err := json.Marshal(d.Products.RetrievalV1) if err != nil { - return nil, fmt.Errorf("marshal source_http_put: %w", err) + return nil, fmt.Errorf("marshal retrievalv1: %w", err) } + ddeal.RetrievalV1 = rev } else { - sourceHttpPutBytes = []byte("null") - } - - // Marshal Format (always present) - formatBytes, err := json.Marshal(d.Data.Format) - if err != nil { - return nil, fmt.Errorf("marshal format: %w", err) + ddeal.RetrievalV1 = []byte("null") } - var spid abi.ActorID - - var ddov1 []byte - if d.Products.DDOV1 != nil { - ddov1, err = json.Marshal(d.Products.DDOV1) - if err != nil { - return nil, fmt.Errorf("marshal ddov1: %w", err) + if d.Products.PDPV1 != nil { + dbpdpv1 := DBPDPV1{ + PDP: d.Products.PDPV1, } - spidInt, err := address.IDFromAddress(d.Products.DDOV1.Provider) + pdpv1, err := json.Marshal(dbpdpv1) if err != nil { - return nil, fmt.Errorf("parse provider address: %w", err) + return nil, fmt.Errorf("marshal pdpv1: %w", err) } - spid = abi.ActorID(spidInt) + ddeal.PDPV1 = pdpv1 } else { - ddov1 = []byte("null") - } - - return &DBDeal{ - Identifier: d.Identifier.String(), - SpID: int64(spid), - PieceCID: d.Data.PieceCID.String(), - Size: int64(d.Data.Size), - Format: formatBytes, - SourceHTTP: sourceHTTPBytes, - SourceAggregate: sourceAggregateBytes, - SourceOffline: sourceOfflineBytes, - SourceHttpPut: sourceHttpPutBytes, - DDOv1: ddov1, - }, nil + ddeal.PDPV1 = []byte("null") + } + + return &ddeal, nil } func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { @@ -383,18 +411,43 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } - n, err := tx.Exec(`INSERT INTO market_mk20_deal (id, sp_id, piece_cid, piece_size, format, source_http, source_aggregate, source_offline, source_http_put, ddo_v1) + n, err := tx.Exec(`INSERT INTO market_mk20_deal (id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data, ddo_v1, retrieval_v1, pdp_v1) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, dbDeal.Identifier, - dbDeal.SpID, + dbDeal.Client, + dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, - dbDeal.Format, - dbDeal.SourceHTTP, - dbDeal.SourceAggregate, - dbDeal.SourceOffline, - dbDeal.SourceHttpPut, - dbDeal.DDOv1) + dbDeal.RawSize, + dbDeal.Data, + dbDeal.DDOv1, + dbDeal.RetrievalV1, + dbDeal.PDPV1) + if err != nil { + return xerrors.Errorf("insert deal: %w", err) + } + if n != 1 { + return xerrors.Errorf("insert deal: expected 1 row affected, got %d", n) + } + return nil +} + +func (d *Deal) UpdateDeal(tx *harmonydb.Tx) error { + dbDeal, err := d.ToDBDeal() + if err != nil { + return xerrors.Errorf("to db deal: %w", err) + } + + n, err := tx.Exec(`UPDATE market_mk20_deal SET + piece_cid_v2 = $1, + piece_cid = $2, + piece_size = $3, + raw_size = $4, + data = $5, + ddo_v1 = $6, + retrieval_v1 = $7, + pdp_v1 = $8`, dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, dbDeal.RawSize, + dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) if err != nil { return xerrors.Errorf("insert deal: %w", err) } @@ -407,16 +460,12 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal err := tx.Select(&dbDeal, `SELECT - id, - piece_cid, - piece_size, - format, - source_http, - source_aggregate, - source_offline, - source_http_put, - ddo_v1, - error FROM market_mk20_deal WHERE id = $1`, id.String()) + id, + client, + data, + ddo_v1, + retrieval_v1, + pdp_v1 FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -429,16 +478,12 @@ func DealFromTX(tx *harmonydb.Tx, id ulid.ULID) (*Deal, error) { func DealFromDB(ctx context.Context, db *harmonydb.DB, id ulid.ULID) (*Deal, error) { var dbDeal []DBDeal err := db.Select(ctx, &dbDeal, `SELECT - id, - piece_cid, - piece_size, - format, - source_http, - source_aggregate, - source_offline, - source_http_put, + id, + client, + data, ddo_v1, - error FROM market_mk20_deal WHERE id = $1`, id.String()) + retrieval_v1, + pdp_v1 FROM market_mk20_deal WHERE id = $1`, id.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } @@ -449,73 +494,53 @@ func DealFromDB(ctx context.Context, db *harmonydb.DB, id ulid.ULID) (*Deal, err } func (d *DBDeal) ToDeal() (*Deal, error) { - var ds DataSource - var products Products - - // Unmarshal each field into the corresponding sub-structs (nil will remain nil if json is "null" or empty) - if err := json.Unmarshal(d.Format, &ds.Format); err != nil { - return nil, fmt.Errorf("unmarshal format: %w", err) - } - - if len(d.SourceHTTP) > 0 && string(d.SourceHTTP) != "null" { - var sh DataSourceHTTP - if err := json.Unmarshal(d.SourceHTTP, &sh); err != nil { - return nil, fmt.Errorf("unmarshal source_http: %w", err) - } - ds.SourceHTTP = &sh - } + var deal Deal - if len(d.SourceAggregate) > 0 && string(d.SourceAggregate) != "null" { - var sa DataSourceAggregate - if err := json.Unmarshal(d.SourceAggregate, &sa); err != nil { - return nil, fmt.Errorf("unmarshal source_aggregate: %w", err) + if len(d.Data) > 0 && string(d.Data) != "null" { + var ds DataSource + if err := json.Unmarshal(d.Data, &ds); err != nil { + return nil, fmt.Errorf("unmarshal data: %w", err) } - ds.SourceAggregate = &sa + deal.Data = &ds } - if len(d.SourceOffline) > 0 && string(d.SourceOffline) != "null" { - var so DataSourceOffline - if err := json.Unmarshal(d.SourceOffline, &so); err != nil { - return nil, fmt.Errorf("unmarshal source_offline: %w", err) + if len(d.DDOv1) > 0 && string(d.DDOv1) != "null" { + var dddov1 DBDDOV1 + if err := json.Unmarshal(d.DDOv1, &dddov1); err != nil { + return nil, fmt.Errorf("unmarshal ddov1: %w", err) } - ds.SourceOffline = &so + deal.Products.DDOV1 = dddov1.DDO } - if len(d.SourceHttpPut) > 0 && string(d.SourceHttpPut) != "null" { - var shp DataSourceHttpPut - if err := json.Unmarshal(d.SourceHttpPut, &shp); err != nil { - return nil, fmt.Errorf("unmarshal source_http_put: %w", err) + if len(d.RetrievalV1) > 0 && string(d.RetrievalV1) != "null" { + var rev RetrievalV1 + if err := json.Unmarshal(d.RetrievalV1, &rev); err != nil { + return nil, fmt.Errorf("unmarshal retrievalv1: %w", err) } - ds.SourceHttpPut = &shp + deal.Products.RetrievalV1 = &rev } - if len(d.DDOv1) > 0 && string(d.DDOv1) != "null" { - if err := json.Unmarshal(d.DDOv1, &products.DDOV1); err != nil { - return nil, fmt.Errorf("unmarshal ddov1: %w", err) + if len(d.PDPV1) > 0 && string(d.PDPV1) != "null" { + var dddov1 DBPDPV1 + if err := json.Unmarshal(d.PDPV1, &dddov1); err != nil { + return nil, fmt.Errorf("unmarshal pdpv1: %w", err) } + deal.Products.PDPV1 = dddov1.PDP } - // Convert identifier id, err := ulid.Parse(d.Identifier) if err != nil { - return nil, fmt.Errorf("parse identifier: %w", err) + return nil, fmt.Errorf("parse id: %w", err) } + deal.Identifier = id - // Convert CID - c, err := cid.Decode(d.PieceCID) + client, err := address.NewFromString(d.Client) if err != nil { - return nil, fmt.Errorf("decode piece_cid: %w", err) + return nil, fmt.Errorf("parse client: %w", err) } + deal.Client = client - // Assign remaining fields - ds.PieceCID = c - ds.Size = abi.PaddedPieceSize(d.Size) - - return &Deal{ - Identifier: id, - Data: ds, - Products: products, - }, nil + return &deal, nil } func DBDealsToDeals(deals []*DBDeal) ([]*Deal, error) { diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index 73084c213..c32aa3299 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -131,13 +131,11 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask var have, missing int64 for p, cents := range toCheck { - commp, err := commcidv2.CommPFromPieceInfo(p) + pieceCid, err := commcidv2.PieceCidV2FromV1(p.PieceCID, uint64(cents[0].RawSize)) if err != nil { return xerrors.Errorf("getting piece commP: %w", err) } - pieceCid := commp.PCidV2() - // Check if the pieceV2 is present in the index store hasEnt, err := c.indexStore.CheckHasPiece(ctx, pieceCid) if err != nil { @@ -145,6 +143,7 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask } if hasEnt { + fmt.Println("Piece cid v2 present in index store") have++ continue } @@ -300,9 +299,9 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask return xerrors.Errorf("parsing provider address: %w", err) } - rawSize, err := deal.Data.RawSize() + pi, err := deal.PieceInfo() if err != nil { - return xerrors.Errorf("getting raw size: %w", err) + return xerrors.Errorf("getting piece info: %w", err) } if uint64(cent.SPID) != spid { @@ -322,31 +321,48 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask aggregation = int(data.Format.Aggregate.Type) } - n, err := c.db.Exec(ctx, `INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, piece_size, raw_size, + var added bool + + _, err = c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, offline, url, indexing, announce, duration, piece_aggregation, started, downloaded, after_commp, aggregated, sector, reg_seal_proof, sector_offset, sealed, indexing_created_at, complete) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, - TRUE, TRUE, True, True, $14, 0, $15 TRUE, NOW(), TRUE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created - deal.Identifier.String(), spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(rawSize), - false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, - cent.SectorID, cent.PieceOff) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + TRUE, TRUE, TRUE, TRUE, $15, 0, $16, TRUE, NOW(), TRUE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created + deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, int64(pi.RawSize), + false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, + cent.SectorID, cent.PieceOff) + if err != nil { + return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + + added = true + + _, err = tx.Exec(`UPDATE market_piece_metadata SET indexed = FALSE WHERE piece_cid = $1 AND piece_size = $2`, p.PieceCID.String(), p.Size) + if err != nil { + return false, xerrors.Errorf("updating market_piece_metadata.indexed column: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) if err != nil { - return xerrors.Errorf("inserting mk20 pipeline: %w", err) + return xerrors.Errorf("inserting into market_mk20_pipeline: %w", err) } - if n != 1 { - return xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + + if added { + log.Infow("added reindexing pipeline entry", "id", id, "task", taskID, "piece", pieceCid) + ongoingIndexingTasks++ + scheduled = true } - log.Infow("added reindexing pipeline entry", "id", id, "task", taskID, "piece", pieceCid) - ongoingIndexingTasks++ - scheduled = true } if scheduled { break // Break out of PieceDeal loop } - } if ongoingIndexingTasks >= int64(MaxOngoingIndexingTasks) { @@ -558,18 +574,21 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask } err = c.db.Select(ctx, &ids, `SELECT m.id - FROM market_mk20_deal AS m - LEFT JOIN ipni AS i - ON m.piece_cid = i.piece_cid - AND m.piece_size = i.piece_size - LEFT JOIN market_mk20_pipeline AS p - ON m.id = p.id - LEFT JOIN market_mk20_pipeline_waiting AS w - ON m.id = w.id - WHERE m.ddo_v1->>'announce_to_ipni' = 'true' - AND i.piece_cid IS NULL - AND p.id IS NULL - AND w.id IS NULL;`) + FROM market_mk20_deal AS m + LEFT JOIN ipni AS i + ON m.piece_cid = i.piece_cid + AND m.piece_size = i.piece_size + LEFT JOIN market_mk20_pipeline AS p + ON m.id = p.id + LEFT JOIN market_mk20_pipeline_waiting AS w + ON m.id = w.id + WHERE m.piece_cid_v2 IS NOT NULL + AND m.ddo_v1 IS NOT NULL + AND m.ddo_v1 != 'null' + AND m.retrieval_v1->>'announce_payload' = 'true' + AND i.piece_cid IS NULL + AND p.id IS NULL + AND w.id IS NULL;`) if err != nil { return xerrors.Errorf("getting mk20 deals which are not announced: %w", err) } @@ -619,17 +638,15 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask return xerrors.Errorf("parsing provider address: %w", err) } - pi := abi.PieceInfo{ - PieceCID: deal.Data.PieceCID, - Size: deal.Data.Size, - } - - commp, err := commcidv2.CommPFromPieceInfo(pi) + pinfo, err := deal.PieceInfo() if err != nil { - return xerrors.Errorf("getting commp from PieceInfo: %w", err) + return xerrors.Errorf("getting piece info: %w", err) } - pcid := commp.PCidV2() + pi := abi.PieceInfo{ + PieceCID: pinfo.PieceCIDV1, + Size: pinfo.Size, + } var ctxIdBuf bytes.Buffer err = pi.MarshalCBOR(&ctxIdBuf) @@ -642,7 +659,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask provider, ok := spToPeer[int64(spid)] if !ok { issues++ - log.Warnw("no peer id for spid", "spid", spid, "checkPiece", pcid) + log.Warnw("no peer id for spid", "spid", spid, "checkPiece", deal.Data.PieceCID.String()) continue } @@ -657,12 +674,12 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask continue } - hasIndex, err := c.indexStore.CheckHasPiece(ctx, pcid) + hasIndex, err := c.indexStore.CheckHasPiece(ctx, deal.Data.PieceCID) if err != nil { return xerrors.Errorf("getting piece hash range: %w", err) } if !hasIndex { - log.Warnw("no index for piece with missing IPNI Ad", "piece", pcid, "checkPiece", pi.PieceCID) + log.Warnw("no index for piece with missing IPNI Ad", "piece", deal.Data.PieceCID.String(), "checkPiece", pi.PieceCID) issues++ continue } @@ -678,7 +695,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask return xerrors.Errorf("getting source sector: %w", err) } if len(sourceSector) == 0 { - log.Warnw("no source sector for piece", "piece", pcid, "checkPiece", pi.PieceCID) + log.Warnw("no source sector for piece", "piece", deal.Data.PieceCID.String(), "checkPiece", pi.PieceCID) issues++ continue } @@ -686,16 +703,11 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask src := sourceSector[0] if !src.PieceRef.Valid { - log.Warnw("no piece ref for ipni reindexing", "piece", pi.PieceCID, "checkPiece", pcid) + log.Warnw("no piece ref for ipni reindexing", "piece", pi.PieceCID, "checkPiece", deal.Data.PieceCID.String()) missing++ continue } - rawSize, err := deal.Data.RawSize() - if err != nil { - return xerrors.Errorf("getting raw size: %w", err) - } - pieceIDUrl := url.URL{ Scheme: "pieceref", Opaque: fmt.Sprintf("%d", src.PieceRef.Int64), @@ -710,14 +722,14 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask } n, err := c.db.Exec(ctx, `INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, piece_size, raw_size, + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, offline, url, indexing, announce, duration, piece_aggregation, started, downloaded, after_commp, aggregated, sector, reg_seal_proof, sector_offset, sealed, indexing_created_at, indexed, complete) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, - TRUE, TRUE, True, True, $14, 0, $15, TRUE, NOW(), TRUE, FALSE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created - deal.Identifier.String(), spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, int64(rawSize), - false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + TRUE, TRUE, TRUE, TRUE, $15, 0, $16, TRUE, NOW(), TRUE, FALSE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created + deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pinfo.PieceCIDV1.String(), pinfo.Size, int64(pinfo.RawSize), + false, pieceIDUrl.String(), true, true, ddo.Duration, aggregation, src.SectorNum, src.PieceOffset) if err != nil { return xerrors.Errorf("inserting mk20 pipeline: %w", err) @@ -782,7 +794,8 @@ func (c *CheckIndexesTask) TypeDetails() harmonytask.TaskTypeDetails { Gpu: 0, Ram: 32 << 20, }, - IAmBored: harmonytask.SingletonTaskAdder(CheckIndexInterval, c), + IAmBored: harmonytask.SingletonTaskAdder(CheckIndexInterval, c), + MaxFailures: 3, } } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index b4491c5c5..a30441595 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -11,7 +11,6 @@ import ( "runtime" "sort" "strconv" - "strings" "sync" "time" @@ -156,13 +155,13 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do // Check if piece is already indexed var indexed bool - err = i.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1`, task.PieceCid).Scan(&indexed) - if err != nil && err != pgx.ErrNoRows { + err = i.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1 and piece_size = $2`, task.PieceCid, task.Size).Scan(&indexed) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("checking if piece %s is already indexed: %w", task.PieceCid, err) } var byteData bool - var subPieces []mk20.PieceDataFormat + var subPieces []mk20.DataSource if task.Mk20 { id, err := ulid.Parse(task.UUID) @@ -220,11 +219,10 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return false, xerrors.Errorf("parsing piece CID: %w", err) } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: pieceCid, Size: task.Size}) + pc2, err := commcidv2.PieceCidV2FromV1(pieceCid, uint64(task.RawSize)) if err != nil { return false, xerrors.Errorf("getting piece commP: %w", err) } - pc2 := commp.PCidV2() var reader storiface.Reader @@ -267,7 +265,7 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do return i.indexStore.AddIndex(ctx, pc2, recs) }) - var aggidx map[cid.Cid][]datasegment.SegmentDesc + var aggidx map[cid.Cid][]indexstore.Record if task.Mk20 && len(subPieces) > 0 { blocks, aggidx, interrupted, err = IndexAggregate(pc2, reader, task.Size, subPieces, recs, addFail) @@ -296,21 +294,8 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do // Save aggregate index if present for k, v := range aggidx { - var idxrecs []indexstore.Record - for _, r := range v { - pi := abi.PieceInfo{PieceCID: r.PieceCID(), Size: abi.PaddedPieceSize(r.Size)} - idxcommp, err := commcidv2.CommPFromPieceInfo(pi) - if err != nil { - return false, xerrors.Errorf("getting piece commP: %w", err) - } - idxrecs = append(idxrecs, indexstore.Record{ - Cid: idxcommp.PCidV2(), - Offset: r.UnpaddedOffest(), - Size: r.UnpaddedLength(), - }) - } - if len(idxrecs) > 0 { - err = i.indexStore.InsertAggregateIndex(ctx, k, idxrecs) + if len(v) > 0 { + err = i.indexStore.InsertAggregateIndex(ctx, k, v) if err != nil { return false, xerrors.Errorf("inserting aggregate index: %w", err) } @@ -483,10 +468,10 @@ type IndexReader interface { func IndexAggregate(pieceCid cid.Cid, reader IndexReader, size abi.PaddedPieceSize, - subPieces []mk20.PieceDataFormat, + subPieces []mk20.DataSource, recs chan<- indexstore.Record, addFail <-chan struct{}, -) (int64, map[cid.Cid][]datasegment.SegmentDesc, bool, error) { +) (int64, map[cid.Cid][]indexstore.Record, bool, error) { dsis := datasegment.DataSegmentIndexStartOffset(size) if _, err := reader.Seek(int64(dsis), io.SeekStart); err != nil { return 0, nil, false, xerrors.Errorf("seeking to data segment index start offset: %w", err) @@ -505,18 +490,16 @@ func IndexAggregate(pieceCid cid.Cid, return 0, nil, false, xerrors.New("no valid data segment index entries") } - aggidx := make(map[cid.Cid][]datasegment.SegmentDesc) - aggidx[pieceCid] = valid + aggidx := make(map[cid.Cid][]indexstore.Record) log.Infow("Indexing aggregate", "piece_size", size, "num_chunks", len(valid), "num_sub_pieces", len(subPieces)) - var haveSubPieces bool - - if len(subPieces) > 0 { + if len(subPieces) > 1 { if len(valid) != len(subPieces) { return 0, nil, false, xerrors.Errorf("expected %d data segment index entries, got %d", len(subPieces), len(idata.Entries)) } - haveSubPieces = true + } else { + return 0, nil, false, xerrors.Errorf("expected at least 2 sub pieces, got 0") } var totalBlocks int64 @@ -528,46 +511,53 @@ func IndexAggregate(pieceCid cid.Cid, strt := entry.UnpaddedOffest() leng := entry.UnpaddedLength() sectionReader := io.NewSectionReader(reader, int64(strt), int64(leng)) - pi := abi.PieceInfo{PieceCID: entry.PieceCID(), Size: abi.PaddedPieceSize(entry.Size)} - commp, err := commcidv2.CommPFromPieceInfo(pi) - if err != nil { - return 0, nil, false, xerrors.Errorf("getting piece commP: %w", err) - } + sp := subPieces[j] - var idx map[cid.Cid][]datasegment.SegmentDesc + //pi := abi.PieceInfo{PieceCID: entry.PieceCID(), Size: abi.PaddedPieceSize(entry.Size)} + //commp, err := commcidv2.CommPFromPieceInfo(pi) + //if err != nil { + // return 0, nil, false, xerrors.Errorf("getting piece commP: %w", err) + //} - b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) - totalBlocks += b + //var idx map[cid.Cid][]datasegment.SegmentDesc + b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) if err != nil { - if strings.Contains(err.Error(), "invalid car version") { - if haveSubPieces { - if subPieces[j].Car != nil { - return 0, aggidx, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) - } - if subPieces[j].Raw != nil { - continue - } - if subPieces[j].Aggregate != nil { - b, idx, inter, err = IndexAggregate(commp.PCidV2(), sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) - if err != nil { - return totalBlocks, aggidx, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) - } - totalBlocks += b - for k, v := range idx { - aggidx[k] = append(aggidx[k], v...) - } - } - } else { - continue - } - } + //// Allow one more layer of aggregation to be indexed + //if strings.Contains(err.Error(), "invalid car version") { + // if haveSubPieces { + // if subPieces[j].Car != nil { + // return 0, aggidx, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) + // } + // if subPieces[j].Raw != nil { + // continue + // } + // if subPieces[j].Aggregate != nil { + // b, idx, inter, err = IndexAggregate(commp.PCidV2(), sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) + // if err != nil { + // return totalBlocks, aggidx, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) + // } + // totalBlocks += b + // for k, v := range idx { + // aggidx[k] = append(aggidx[k], v...) + // } + // } + // } else { + // continue + // } + //} return totalBlocks, aggidx, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) } if inter { return totalBlocks, aggidx, true, nil } + totalBlocks += b + aggidx[pieceCid] = append(aggidx[pieceCid], indexstore.Record{ + Cid: sp.PieceCID, + Offset: strt, + Size: leng, + }) } return totalBlocks, aggidx, false, nil @@ -583,8 +573,8 @@ func (i *IndexingTask) recordCompletion(ctx context.Context, task itask, taskID return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) } } else { - _, err := i.db.Exec(ctx, `SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - task.UUID, task.PieceCid, !task.IsDDO, task.SpID, task.Sector, task.Offset, task.Size, task.RawSize, indexed, false, task.ChainDealId) + _, err := i.db.Exec(ctx, `SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + task.UUID, task.PieceCid, !task.IsDDO, task.SpID, task.Sector, task.Offset, task.Size, task.RawSize, indexed, nil, false, task.ChainDealId) if err != nil { return xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", task.UUID, err) } diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 1801882a5..ebf9081a7 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -119,9 +119,15 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, xerrors.Errorf("unmarshaling piece info: %w", err) } - commp, err := commcidv2.CommPFromPieceInfo(pi) + var rawSize abi.UnpaddedPieceSize + err = I.db.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1`, pi.PieceCID.String(), pi.Size).Scan(&rawSize) if err != nil { - return false, xerrors.Errorf("getting piece commP: %w", err) + return false, xerrors.Errorf("querying raw size: %w", err) + } + + pcid2, err := commcidv2.PieceCidV2FromV1(pi.PieceCID, uint64(rawSize)) + if err != nil { + return false, xerrors.Errorf("getting piece CID v2: %w", err) } // Try to read unsealed sector first (mk12 deal) @@ -135,7 +141,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b if err != nil { serr := err // Try to read piece (mk20 deal) - reader, _, err = I.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) + reader, _, err = I.cpr.GetSharedPieceReader(ctx, pcid2) if err != nil { return false, xerrors.Errorf("getting piece reader from sector and piece park: %w, %w", serr, err) } @@ -162,7 +168,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b var eg errgroup.Group addFail := make(chan struct{}) var interrupted bool - var subPieces []mk20.PieceDataFormat + var subPieces []mk20.DataSource chk := chunker.NewInitialChunker() eg.Go(func() error { @@ -198,7 +204,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { subPieces = deal.Data.Format.Aggregate.Sub - _, _, interrupted, err = IndexAggregate(commp.PCidV2(), reader, pi.Size, subPieces, recs, addFail) + _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.Size, subPieces, recs, addFail) } } @@ -228,7 +234,7 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, nil } - lnk, err := chk.Finish(ctx, I.db, commp.PCidV2()) + lnk, err := chk.Finish(ctx, I.db, pcid2) if err != nil { return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) } diff --git a/tasks/pdp/task_addroot.go b/tasks/pdp/task_addroot.go new file mode 100644 index 000000000..c9b25a42a --- /dev/null +++ b/tasks/pdp/task_addroot.go @@ -0,0 +1,190 @@ +package pdp + +//import ( +// "context" +// "database/sql" +// "errors" +// "math/big" +// "net/http" +// "time" +// +// "github.com/ethereum/go-ethereum/common" +// "github.com/ethereum/go-ethereum/core/types" +// "github.com/ethereum/go-ethereum/ethclient" +// "github.com/filecoin-project/curio/harmony/harmonydb" +// "github.com/filecoin-project/curio/harmony/harmonytask" +// "github.com/filecoin-project/curio/harmony/resources" +// "github.com/filecoin-project/curio/harmony/taskhelp" +// "github.com/filecoin-project/curio/lib/passcall" +// "github.com/filecoin-project/curio/pdp/contract" +// "github.com/filecoin-project/curio/tasks/message" +// types2 "github.com/filecoin-project/lotus/chain/types" +// "golang.org/x/xerrors" +//) +// +//type PDPServiceNodeApi interface { +// ChainHead(ctx context.Context) (*types2.TipSet, error) +//} +// +//type PDPTaskAddRoot struct { +// db *harmonydb.DB +// sender *message.SenderETH +// ethClient *ethclient.Client +// filClient PDPServiceNodeApi +//} +// +//func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +// ctx := context.Background() +// +// // Step 5: Prepare the Ethereum transaction data outside the DB transaction +// // Obtain the ABI of the PDPVerifier contract +// abiData, err := contract.PDPVerifierMetaData.GetAbi() +// if err != nil { +// return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) +// } +// +// // Prepare RootData array for Ethereum transaction +// // Define a Struct that matches the Solidity RootData struct +// type RootData struct { +// Root struct{ Data []byte } +// RawSize *big.Int +// } +// +// var rootDataArray []RootData +// +// rootData := RootData{ +// Root: struct{ Data []byte }{Data: rootCID.Bytes()}, +// RawSize: new(big.Int).SetUint64(totalSize), +// } +// +// // Step 6: Prepare the Ethereum transaction +// // Pack the method call data +// // The extraDataBytes variable is now correctly populated above +// data, err := abiData.Pack("addRoots", proofSetID, rootDataArray, extraDataBytes) +// if err != nil { +// return false, xerrors.Errorf("packing data: %w", err) +// } +// +// // Step 7: Get the sender address from 'eth_keys' table where role = 'pdp' limit 1 +// fromAddress, err := p.getSenderAddress(ctx) +// if err != nil { +// return false, xerrors.Errorf("getting sender address: %w", err) +// } +// +// // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) +// txEth := types.NewTransaction( +// 0, +// contract.ContractAddresses().PDPVerifier, +// big.NewInt(0), +// 0, +// nil, +// data, +// ) +// +// // Step 8: Send the transaction using SenderETH +// reason := "pdp-addroots" +// txHash, err := p.sender.Send(ctx, fromAddress, txEth, reason) +// if err != nil { +// return false, xerrors.Errorf("sending transaction: %w", err) +// } +// +// // Step 9: Insert into message_waits_eth and pdp_proofset_roots +// _, err = p.db.BeginTransaction(ctx, func(txdb *harmonydb.Tx) (bool, error) { +// // Insert into message_waits_eth +// _, err = txdb.Exec(` +// INSERT INTO message_waits_eth (signed_tx_hash, tx_status) +// VALUES ($1, $2) +// `, txHash.Hex(), "pending") +// if err != nil { +// return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) +// } +// +// // Update proof set for initialization upon first add +// _, err = txdb.Exec(` +// UPDATE pdp_proof_sets SET init_ready = true +// WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL +// `, proofSetIDUint64) +// if err != nil { +// return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) +// } +// +// // Insert into pdp_proofset_roots +// +// for addMessageIndex, addRootReq := range payload.Roots { +// for _, subrootEntry := range addRootReq.Subroots { +// subrootInfo := subrootInfoMap[subrootEntry.SubrootCID] +// +// // Insert into pdp_proofset_roots +// _, err = txdb.Exec(` +// INSERT INTO pdp_proofset_root_adds ( +// proofset, +// root, +// add_message_hash, +// add_message_index, +// subroot, +// subroot_offset, +// subroot_size, +// pdp_pieceref +// ) +// VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +// `, +// proofSetIDUint64, +// addRootReq.RootCID, +// txHash.Hex(), +// addMessageIndex, +// subrootEntry.SubrootCID, +// subrootInfo.SubrootOffset, +// subrootInfo.PieceInfo.Size, +// subrootInfo.PDPPieceRefID, +// ) +// if err != nil { +// return false, err +// } +// } +// } +// +// // Return true to commit the transaction +// return true, nil +// }, harmonydb.OptionRetry()) +// if err != nil { +// return false, xerrors.Errorf("failed to save details to DB: %w", err) +// } +// return true, nil +//} +// +//// getSenderAddress retrieves the sender address from the database where role = 'pdp' limit 1 +//func (p *PDPTaskAddRoot) getSenderAddress(ctx context.Context) (common.Address, error) { +// var addressStr string +// err := p.db.QueryRow(ctx, `SELECT address FROM eth_keys WHERE role = 'pdp' LIMIT 1`).Scan(&addressStr) +// if err != nil { +// if errors.Is(err, sql.ErrNoRows) { +// return common.Address{}, errors.New("no sender address with role 'pdp' found") +// } +// return common.Address{}, err +// } +// address := common.HexToAddress(addressStr) +// return address, nil +//} +// +//func (p *PDPTaskAddRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +// return &ids[0], nil +//} +// +//func (p *PDPTaskAddRoot) TypeDetails() harmonytask.TaskTypeDetails { +// return harmonytask.TaskTypeDetails{ +// Max: taskhelp.Max(50), +// Name: "PDPAddRoot", +// Cost: resources.Resources{ +// Cpu: 1, +// Ram: 64 << 20, +// }, +// MaxFailures: 3, +// IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { +// return p.schedule(context.Background(), taskFunc) +// }), +// } +//} +// +//func (p *PDPTaskAddRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} +// +//var _ harmonytask.TaskInterface = &PDPTaskAddRoot{} diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index b0699de68..9a29f94db 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -5,11 +5,9 @@ import ( "errors" "fmt" "io" - "os" + "net/url" "time" - "github.com/filecoin-project/curio/lib/ffi" - "github.com/filecoin-project/curio/lib/storiface" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/oklog/ulid" @@ -17,30 +15,28 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" - "github.com/filecoin-project/curio/lib/dealdata" + "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/mk20" ) type AggregateChunksTask struct { db *harmonydb.DB - stor paths.StashStore remote *paths.Remote sc *ffi.SealCalls } -func NewAggregateChunksTask(db *harmonydb.DB, stor paths.StashStore, remote *paths.Remote, sc *ffi.SealCalls) *AggregateChunksTask { +func NewAggregateChunksTask(db *harmonydb.DB, remote *paths.Remote, sc *ffi.SealCalls) *AggregateChunksTask { return &AggregateChunksTask{ db: db, - stor: stor, remote: remote, sc: sc, } @@ -93,7 +89,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } var rawSize int64 - var pcid cid.Cid + var pcid, pcid2 cid.Cid var psize abi.PaddedPieceSize var deal *mk20.Deal @@ -102,13 +98,14 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo if err != nil { return false, xerrors.Errorf("getting deal details: %w", err) } - raw, err := deal.Data.RawSize() + pi, err := deal.PieceInfo() if err != nil { - return false, xerrors.Errorf("getting deal raw size: %w", err) + return false, xerrors.Errorf("getting piece info: %w", err) } - rawSize = int64(raw) - pcid = deal.Data.PieceCID - psize = deal.Data.Size + rawSize = int64(pi.RawSize) + pcid = pi.PieceCIDV1 + psize = pi.Size + pcid2 = deal.Data.PieceCID } else { rawSize = 4817498192 // TODO: Fix this for PDP fmt.Println(uid) @@ -147,85 +144,91 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo rd := io.MultiReader(readers...) - w := &writer.Writer{} + var parkedPieceID, pieceRefID int64 + var pieceParked bool - // Function to write data into StashStore and calculate commP - writeFunc := func(f *os.File) error { - limitReader := io.LimitReader(rd, rawSize) - - multiWriter := io.MultiWriter(w, f) - - n, err := io.CopyBuffer(multiWriter, limitReader, make([]byte, writer.CommPBuf)) + comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, + pcid.String(), psize, rawSize).Scan(&parkedPieceID, &pieceParked) if err != nil { - return fmt.Errorf("failed to read and write aggregated piece data: %w", err) + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) } - if n != rawSize { - return fmt.Errorf("number of bytes written to CommP writer %d not equal to the file size %d", n, rawSize) + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, "/PUT").Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) } - return nil - } - - stashID, err := a.stor.StashCreate(ctx, rawSize, writeFunc) - if err != nil { - return false, xerrors.Errorf("stashing aggregated piece data: %w", err) - } - - calculatedCommp, err := w.Sum() + return true, nil + }, harmonydb.OptionRetry()) if err != nil { - return false, xerrors.Errorf("computing commP failed: %w", err) - } - - if !calculatedCommp.PieceCID.Equals(pcid) { - return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", calculatedCommp.PieceCID.String(), pcid.String()) + return false, xerrors.Errorf("saving aggregated chunk details to DB: %w", err) } - if calculatedCommp.PieceSize != psize { - return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", calculatedCommp.PieceSize, psize) + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") } - stashUrl, err := a.stor.StashURL(stashID) - if err != nil { - return false, xerrors.Errorf("getting stash URL: %w", err) - } - stashUrl.Scheme = dealdata.CustoreScheme + failed := true + var cleanupChunks bool - comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - var parkedPieceID int64 + // Clean up piece park tables in case of failure + // TODO: Figure out if there is a race condition with cleanup task + defer func() { + if cleanupChunks { + _, serr := a.db.Exec(ctx, `DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) + if serr != nil { + log.Errorf("failed to delete market_mk20_deal_chunk entry: %w", serr) + } + _, serr = a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) + if serr != nil { + log.Errorf("failed to delete parked_piece_refs entry: %w", serr) + } + } + if failed { + _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) + if err != nil { + log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) + } + } + }() - err = tx.QueryRow(` - INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - VALUES ($1, $2, $3, TRUE) RETURNING id - `, calculatedCommp.PieceCID.String(), calculatedCommp.PieceSize, rawSize).Scan(&parkedPieceID) + // Write piece if not already complete + if !pieceParked { + pi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), rawSize, rd, storiface.PathStorage) if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) } - var pieceRefID int64 - err = tx.QueryRow(` - INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, TRUE) RETURNING ref_id - `, parkedPieceID, stashUrl.String()).Scan(&pieceRefID) - if err != nil { - return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + if !pi.PieceCID.Equals(pcid) { + cleanupChunks = true + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", pi.PieceCID.String(), pcid.String()) } - if isMk20 { - n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - id.String(), deal.Data.PieceCID.String(), deal.Data.Size, []int64{pieceRefID}) - if err != nil { - return false, xerrors.Errorf("inserting mk20 download pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting mk20 download pipeline: %d rows affected", n) - } + if pi.Size != psize { + cleanupChunks = true + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", pi.Size, psize) + } + } + // Update DB status of piece, deal, PDP + comm, err = a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + if isMk20 { spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) if err != nil { return false, fmt.Errorf("getting provider ID: %w", err) } + var rev mk20.RetrievalV1 + if deal.Products.RetrievalV1 != nil { + rev = *deal.Products.RetrievalV1 + } + ddo := deal.Products.DDOV1 dealdata := deal.Data dealID := deal.Identifier.String() @@ -242,14 +245,29 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo aggregation = int(dealdata.Format.Aggregate.Type) } - n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, - piece_size, raw_size, offline, indexing, announce, - allocation_id, duration, piece_aggregation, started, after_commp) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE, TRUE)`, - dealID, spid, ddo.ContractAddress, ddo.Client.String(), dealdata.PieceCID.String(), - dealdata.Size, int64(dealdata.SourceHttpPut.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, - allocationID, ddo.Duration, aggregation) + if !pieceParked { + _, err = tx.Exec(`UPDATE parked_pieces SET + complete = TRUE + WHERE id = $1 + AND complete = false`, pieceRefID) + if err != nil { + return false, xerrors.Errorf("marking piece park as complete: %w", err) + } + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", pieceRefID), + } + + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid_v2, piece_cid, + piece_size, raw_size, url, offline, indexing, announce, + allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, + dealID, spid, ddo.ContractAddress, deal.Client.String(), pcid2.String(), pcid.String(), + psize, rawSize, pieceIDUrl.String(), false, rev.Indexing, rev.AnnouncePayload, + allocationID, ddo.Duration, aggregation, aggregation) if err != nil { return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) } @@ -275,16 +293,18 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo return false, xerrors.Errorf("not implemented for PDP") // TODO: Do what is required for PDP } - return true, nil }, harmonydb.OptionRetry()) + if err != nil { - return false, xerrors.Errorf("saving aggregated chunk details to DB: %w", err) + return false, xerrors.Errorf("updating DB: %w", err) } - if !comm { return false, xerrors.Errorf("failed to commit the transaction") } + + failed = false + return true, nil } @@ -300,7 +320,7 @@ func (a *AggregateChunksTask) TypeDetails() harmonytask.TaskTypeDetails { Cpu: 1, Ram: 4 << 30, }, - MaxFailures: 3, + MaxFailures: 1, IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { return a.schedule(context.Background(), taskFunc) }), diff --git a/tasks/piece/task_park_piece.go b/tasks/piece/task_park_piece.go index 08f8a6175..010ff13a8 100644 --- a/tasks/piece/task_park_piece.go +++ b/tasks/piece/task_park_piece.go @@ -76,6 +76,7 @@ func (p *ParkPieceTask) pollPieceTasks(ctx context.Context) { FROM parked_pieces WHERE long_term = $1 AND complete = FALSE + AND skip = FALSE AND task_id IS NULL `, p.longTerm) if err != nil { @@ -96,7 +97,7 @@ func (p *ParkPieceTask) pollPieceTasks(ctx context.Context) { p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, err error) { // Update n, err := tx.Exec( - `UPDATE parked_pieces SET task_id = $1 WHERE id = $2 AND complete = FALSE AND task_id IS NULL AND long_term = $3`, + `UPDATE parked_pieces SET task_id = $1 WHERE id = $2 AND complete = FALSE AND skip = FALSE AND task_id IS NULL AND long_term = $3`, id, pieceID.ID, p.longTerm) if err != nil { return false, xerrors.Errorf("updating parked piece: %w", err) diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index ac09a7fb6..23b24c2ad 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -38,6 +38,7 @@ type MK20PipelinePiece struct { SPID int64 `db:"sp_id"` Client string `db:"client"` Contract string `db:"contract"` + PieceCIDV2 string `db:"piece_cid_v2"` PieceCID string `db:"piece_cid"` PieceSize int64 `db:"piece_size"` RawSize int64 `db:"raw_size"` @@ -165,9 +166,17 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 return fmt.Errorf("getting provider ID: %w", err) } + var rev mk20.RetrievalV1 + if deal.Products.RetrievalV1 != nil { + rev = *deal.Products.RetrievalV1 + } ddo := deal.Products.DDOV1 data := deal.Data dealID := deal.Identifier.String() + pi, err := deal.PieceInfo() + if err != nil { + return fmt.Errorf("getting piece info: %w", err) + } var allocationID interface{} if ddo.AllocationId != nil { @@ -185,7 +194,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 if data.SourceHTTP != nil { var pieceID int64 // Attempt to select the piece ID first - err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, data.PieceCID.String(), data.Size).Scan(&pieceID) + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, pi.PieceCIDV1.String(), pi.Size).Scan(&pieceID) if err != nil { if errors.Is(err, pgx.ErrNoRows) { @@ -194,7 +203,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) VALUES ($1, $2, $3, TRUE) ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING - RETURNING id`, data.PieceCID.String(), int64(data.Size), int64(data.SourceHTTP.RawSize)).Scan(&pieceID) + RETURNING id`, pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&pieceID) if err != nil { return xerrors.Errorf("inserting new parked piece and getting id: %w", err) } @@ -224,7 +233,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 } n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - dealID, data.PieceCID.String(), data.Size, refIds) + dealID, pi.PieceCIDV1.String(), pi.Size, refIds) if err != nil { return xerrors.Errorf("inserting mk20 download pipeline: %w", err) } @@ -233,12 +242,12 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 } n, err = tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation, started) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, TRUE)`, - dealID, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), - data.Size, int64(data.SourceHTTP.RawSize), false, ddo.Indexing, ddo.AnnounceToIPNI, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, TRUE)`, + dealID, spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + pi.Size, pi.RawSize, false, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, aggregation) if err != nil { return xerrors.Errorf("inserting mk20 pipeline: %w", err) @@ -252,12 +261,12 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 // INSERT Pipeline when data source is offline if deal.Data.SourceOffline != nil { n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, - dealID, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), - data.Size, int64(data.SourceOffline.RawSize), true, ddo.Indexing, ddo.AnnounceToIPNI, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, + dealID, spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + pi.Size, pi.RawSize, true, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, aggregation) if err != nil { return xerrors.Errorf("inserting mk20 pipeline: %w", err) @@ -281,12 +290,16 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 toDownload := make(map[downloadkey][]mk20.HttpUrl) for _, piece := range deal.Data.SourceAggregate.Pieces { + spi, err := mk20.GetPieceInfo(piece.PieceCID) + if err != nil { + return xerrors.Errorf("getting piece info: %w", err) + } if piece.SourceHTTP != nil { - urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size, RawSize: piece.SourceHTTP.RawSize}] + urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] if ok { - toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size}] = append(urls, piece.SourceHTTP.URLs...) + toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) } else { - toDownload[downloadkey{ID: dealID, PieceCID: piece.PieceCID, Size: piece.Size, RawSize: piece.SourceHTTP.RawSize}] = piece.SourceHTTP.URLs + toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs } } } @@ -352,16 +365,16 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 if piece.SourceOffline != nil { offline = true } - rawSize, err := piece.RawSize() + spi, err := mk20.GetPieceInfo(piece.PieceCID) if err != nil { - return xerrors.Errorf("getting raw size: %w", err) + return xerrors.Errorf("getting piece info: %w", err) } - pBatch.Queue(`INSERT INTO market_mk20_pipeline (id, sp_id, contract, client, piece_cid, + pBatch.Queue(`INSERT INTO market_mk20_pipeline (id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, aggr_index, started) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)`, - dealID, spid, ddo.ContractAddress, ddo.Client.String(), piece.PieceCID.String(), - piece.Size, rawSize, offline, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)`, + dealID, spid, ddo.ContractAddress, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), + spi.Size, spi.RawSize, offline, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, 0, data.Format.Aggregate.Type, i, !offline) if pBatch.Len() > pBatchSize { res := tx.SendBatch(ctx, pBatch) @@ -392,6 +405,7 @@ func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { sp_id, contract, client, + piece_cid_v2, piece_cid, piece_size, raw_size, @@ -532,7 +546,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie // Check if We can find the URL for this piece on remote servers for rUrl, headers := range d.urls { // Create a new HTTP request - urlString := fmt.Sprintf("%s?id=%s", rUrl, piece.PieceCID) + urlString := fmt.Sprintf("%s?id=%s", rUrl, piece.PieceCIDV2) req, err := http.NewRequest(http.MethodHead, urlString, nil) if err != nil { return false, xerrors.Errorf("error creating request: %w", err) diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index 1246c9236..f296c7dc5 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -29,6 +29,7 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/multictladdr" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/promise" @@ -78,7 +79,7 @@ type CurioStorageDealMarket struct { urls map[string]http.Header adders [numPollers]promise.Promise[harmonytask.AddTaskFunc] as *multictladdr.MultiAddressSelector - stor paths.StashStore + sc *ffi.SealCalls } type MK12Pipeline struct { @@ -115,7 +116,7 @@ type MK12Pipeline struct { Offset *int64 `db:"sector_offset"` } -func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector, stor paths.StashStore) *CurioStorageDealMarket { +func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector, sc *ffi.SealCalls) *CurioStorageDealMarket { moduleMap := make(map[string][]address.Address) moduleMap[mk12Str] = miners @@ -135,7 +136,7 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * urls: urls, as: as, ethClient: ethClient, - stor: stor, + sc: sc, } } @@ -186,7 +187,7 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { if len(miners) == 0 { return nil } - d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as, d.stor) + d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as, d.sc) if err != nil { return err } @@ -525,21 +526,36 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea var updated bool err = tx.QueryRow(` WITH selected_data AS ( - SELECT url, headers, raw_size - FROM market_offline_urls - WHERE uuid = $1 + SELECT url, headers, raw_size + FROM market_offline_urls + WHERE uuid = $1 + ), + updated_pipeline AS ( + UPDATE market_mk12_deal_pipeline + SET url = selected_data.url, + headers = selected_data.headers, + raw_size = selected_data.raw_size, + started = TRUE + FROM selected_data + WHERE market_mk12_deal_pipeline.uuid = $1 + RETURNING uuid + ), + updated_deals AS ( + UPDATE market_mk12_deals + SET raw_size = selected_data.raw_size + FROM selected_data + WHERE market_mk12_deals.uuid = $1 + RETURNING uuid + ), + updated_direct_deals AS ( + UPDATE market_direct_deals + SET raw_size = selected_data.raw_size + FROM selected_data + WHERE market_direct_deals.uuid = $1 + RETURNING uuid ) - UPDATE market_mk12_deal_pipeline - SET url = selected_data.url, - headers = selected_data.headers, - raw_size = selected_data.raw_size, - started = TRUE - FROM selected_data - WHERE market_mk12_deal_pipeline.uuid = $1 - RETURNING CASE - WHEN EXISTS (SELECT 1 FROM selected_data) THEN TRUE - ELSE FALSE - END;`, deal).Scan(&updated) + SELECT + (EXISTS (SELECT 1 FROM selected_data)) AS updated;`, deal).Scan(&updated) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("failed to update the pipeline for deal %s: %w", deal, err) @@ -598,6 +614,16 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea return false, xerrors.Errorf("store url for piece %s: updating pipeline: %w", pcid, err) } + _, err = tx.Exec(`UPDATE market_mk12_deals SET raw_size = $1 WHERE uuid = $2`, rawSize, deal) + if err != nil { + return false, xerrors.Errorf("store url for piece %s: updating deals: %w", pcid, err) + } + + _, err = tx.Exec(`UPDATE market_direct_deals SET raw_size = $1 WHERE uuid = $2`, rawSize, deal) + if err != nil { + return false, xerrors.Errorf("store url for piece %s: updating direct deals: %w", pcid, err) + } + return true, nil } return false, nil diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index 1f4b6d36d..86b8aede0 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -6,23 +6,19 @@ import ( "io" "math/bits" "net/url" - "os" "strconv" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "golang.org/x/xerrors" - "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-data-segment/datasegment" - "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" - "github.com/filecoin-project/curio/lib/dealdata" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/storiface" @@ -112,9 +108,9 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d return false, xerrors.Errorf("getting deal details from DB: %w", err) } - rawSize, err := deal.Data.RawSize() + pi, err := deal.PieceInfo() if err != nil { - return false, xerrors.Errorf("getting raw size: %w", err) + return false, xerrors.Errorf("getting piece info: %w", err) } var pinfos []abi.PieceInfo @@ -182,8 +178,7 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d PieceCID: pcid, }) - pReader, _ := padreader.New(reader, uint64(piece.RawSize)) - readers = append(readers, pReader) + readers = append(readers, io.LimitReader(reader, piece.RawSize)) refIDs = append(refIDs, refNum) } @@ -206,74 +201,66 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d return false, xerrors.Errorf("aggregating piece readers: %w", err) } - w := &writer.Writer{} + var parkedPieceID, pieceRefID int64 + var pieceParked bool - // Function to write data into StashStore and calculate commP - writeFunc := func(f *os.File) error { - multiWriter := io.MultiWriter(w, f) - - // Copy data from limitedReader to multiWriter - n, err := io.CopyBuffer(multiWriter, outR, make([]byte, writer.CommPBuf)) + comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, + pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID, &pieceParked) if err != nil { - return fmt.Errorf("failed to read and write aggregated piece data: %w", err) + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) } - if n != int64(rawSize) { - return fmt.Errorf("number of bytes written to CommP writer %d not equal to the file size %d", n, aggregatedRawSize) + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, "/Aggregate").Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) } - return nil - } - - stashID, err := a.stor.StashCreate(ctx, int64(next), writeFunc) - if err != nil { - return false, xerrors.Errorf("stashing aggregated piece data: %w", err) - } - - calculatedCommp, err := w.Sum() + return true, nil + }, harmonydb.OptionRetry()) if err != nil { - return false, xerrors.Errorf("computing commP failed: %w", err) - } - - if !calculatedCommp.PieceCID.Equals(deal.Data.PieceCID) { - return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", calculatedCommp.PieceCID.String(), deal.Data.PieceCID.String()) + return false, xerrors.Errorf("saving aggregated chunk details to DB: %w", err) } - if calculatedCommp.PieceSize != deal.Data.Size { - return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", calculatedCommp.PieceSize, deal.Data.Size) + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") } - comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - var parkedPieceID int64 + failed := true - err = tx.QueryRow(` - INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) - VALUES ($1, $2, $3, TRUE) RETURNING id - `, calculatedCommp.PieceCID.String(), calculatedCommp.PieceSize, rawSize).Scan(&parkedPieceID) - if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + // Clean up piece park tables in case of failure + // TODO: Figure out if there is a race condition with cleanup task + defer func() { + if failed { + _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) + if err != nil { + log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) + } } + }() - // Create a piece ref with data_url being "stashstore://" - // Get StashURL - stashURL, err := a.stor.StashURL(stashID) + // Write piece if not already complete + if !pieceParked { + upi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage) if err != nil { - return false, fmt.Errorf("failed to get stash URL: %w", err) + return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) } - // Change scheme to "custore" - stashURL.Scheme = dealdata.CustoreScheme - dataURL := stashURL.String() + if !upi.PieceCID.Equals(pi.PieceCIDV1) { + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", upi.PieceCID.String(), pi.PieceCIDV1.String()) + } - var pieceRefID int64 - err = tx.QueryRow(` - INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, TRUE) RETURNING ref_id - `, parkedPieceID, dataURL).Scan(&pieceRefID) - if err != nil { - return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + if upi.Size != pi.Size { + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", upi.Size, pi.Size) } + } + comm, err = a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { pieceIDUrl := url.URL{ Scheme: "pieceref", Opaque: fmt.Sprintf("%d", pieceRefID), @@ -290,6 +277,11 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d return false, fmt.Errorf("failed to delete parked_piece_refs entries: %w", err) } + var rev mk20.RetrievalV1 + if deal.Products.RetrievalV1 != nil { + rev = *deal.Products.RetrievalV1 + } + ddo := deal.Products.DDOV1 data := deal.Data @@ -301,12 +293,12 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d } n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid, piece_size, raw_size, url, + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, url, offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, TRUE, TRUE, TRUE, TRUE)`, - id, spid, ddo.ContractAddress, ddo.Client.String(), data.PieceCID.String(), data.Size, rawSize, pieceIDUrl.String(), - false, ddo.Indexing, ddo.AnnounceToIPNI, allocationID, ddo.Duration, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE, TRUE)`, + id, spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), + false, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, data.Format.Aggregate.Type, data.Format.Aggregate.Type) if err != nil { return false, xerrors.Errorf("inserting aggregated piece in mk20 pipeline: %w", err) @@ -323,6 +315,9 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d if !comm { return false, xerrors.Errorf("failed to commit the transaction") } + + failed = false + return true, nil } diff --git a/tasks/storage-market/task_commp.go b/tasks/storage-market/task_commp.go index 11758f285..c5b8607cd 100644 --- a/tasks/storage-market/task_commp.go +++ b/tasks/storage-market/task_commp.go @@ -3,7 +3,6 @@ package storage_market import ( "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -11,7 +10,6 @@ import ( "strconv" "github.com/ipfs/go-cid" - "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-commp-utils/writer" @@ -442,11 +440,11 @@ func checkExpiry(ctx context.Context, db *harmonydb.DB, api headAPI, deal string var starts []struct { StartEpoch int64 `db:"start_epoch"` } - err := db.Select(ctx, &starts, `SELECT start_epoch FROM market_mk12_deals WHERE uuid = $1 LIMIT 1`, deal) + err := db.Select(ctx, &starts, `SELECT start_epoch FROM market_mk12_deals WHERE uuid = $1 + UNION ALL + SELECT start_epoch FROM market_direct_deals WHERE uuid = $1 + LIMIT 1`, deal) if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return false, nil - } return false, xerrors.Errorf("failed to get start epoch from DB: %w", err) } if len(starts) != 1 { diff --git a/web/api/webrpc/deals.go b/web/api/webrpc/deals.go index 7c7f31c33..510fa6c2a 100644 --- a/web/api/webrpc/deals.go +++ b/web/api/webrpc/deals.go @@ -21,6 +21,7 @@ type OpenDealInfo struct { SectorNumber uint64 `db:"sector_number"` PieceCID string `db:"piece_cid"` PieceSize uint64 `db:"piece_size"` + RawSize uint64 `db:"data_raw_size"` CreatedAt time.Time `db:"created_at"` SnapDeals bool `db:"is_snap"` @@ -33,7 +34,7 @@ type OpenDealInfo struct { func (a *WebRPC) DealsPending(ctx context.Context) ([]OpenDealInfo, error) { deals := []OpenDealInfo{} - err := a.deps.DB.Select(ctx, &deals, `SELECT sp_id, sector_number, piece_cid, piece_size, created_at, is_snap FROM open_sector_pieces ORDER BY created_at DESC`) + err := a.deps.DB.Select(ctx, &deals, `SELECT sp_id, sector_number, piece_cid, piece_size, data_raw_size, created_at, is_snap FROM open_sector_pieces ORDER BY created_at DESC`) if err != nil { return nil, err } @@ -50,11 +51,11 @@ func (a *WebRPC) DealsPending(ctx context.Context) ([]OpenDealInfo, error) { if err != nil { return nil, xerrors.Errorf("failed to parse piece cid: %w", err) } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: pcid, Size: abi.PaddedPieceSize(deals[i].PieceSize)}) + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, deals[i].RawSize) if err != nil { return nil, xerrors.Errorf("failed to get commp: %w", err) } - deals[i].PieceCidV2 = commp.PCidV2().String() + deals[i].PieceCidV2 = pcid2.String() } return deals, nil diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index 1c66b43a6..4dd7eb25b 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -100,7 +100,14 @@ func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { return nil, xerrors.Errorf("failed to unmarshal piece info: %w", err) } - commp, err := commcidv2.CommPFromPieceInfo(pi) + // Get RawSize from market_piece_deal to calculate PieceCidV2 + var rawSize uint64 + err = a.deps.DB.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1;`, pi.PieceCID, pi.Size).Scan(&rawSize) + if err != nil { + return nil, xerrors.Errorf("failed to get raw size: %w", err) + } + + pcidv2, err := commcidv2.PieceCidV2FromV1(pi.PieceCID, rawSize) if err != nil { return nil, xerrors.Errorf("failed to get commp: %w", err) } @@ -108,7 +115,7 @@ func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { details.PieceCid = pi.PieceCID.String() size := int64(pi.Size) details.PieceSize = size - details.PieceCidV2 = commp.PCidV2().String() + details.PieceCidV2 = pcidv2.String() maddr, err := address.NewIDAddress(uint64(details.SpID)) if err != nil { diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 78cfbc0b6..90ff63734 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -92,30 +92,30 @@ func (a *WebRPC) SetStorageAsk(ctx context.Context, ask *StorageAsk) error { } type MK12Pipeline struct { - UUID string `db:"uuid" json:"uuid"` - SpID int64 `db:"sp_id" json:"sp_id"` - Started bool `db:"started" json:"started"` - PieceCid string `db:"piece_cid" json:"piece_cid"` - PieceSize int64 `db:"piece_size" json:"piece_size"` - PieceCidV2 string `db:"-" json:"piece_cid_v2"` - RawSize *int64 `db:"raw_size" json:"raw_size"` - Offline bool `db:"offline" json:"offline"` - URL *string `db:"url" json:"url"` - Headers []byte `db:"headers" json:"headers"` - CommTaskID *int64 `db:"commp_task_id" json:"commp_task_id"` - AfterCommp bool `db:"after_commp" json:"after_commp"` - PSDTaskID *int64 `db:"psd_task_id" json:"psd_task_id"` - AfterPSD bool `db:"after_psd" json:"after_psd"` - PSDWaitTime *time.Time `db:"psd_wait_time" json:"psd_wait_time"` - FindDealTaskID *int64 `db:"find_deal_task_id" json:"find_deal_task_id"` - AfterFindDeal bool `db:"after_find_deal" json:"after_find_deal"` - Sector *int64 `db:"sector" json:"sector"` - Offset *int64 `db:"sector_offset" json:"sector_offset"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - Indexed bool `db:"indexed" json:"indexed"` - Announce bool `db:"announce" json:"announce"` - Complete bool `db:"complete" json:"complete"` - Miner string `json:"miner"` + UUID string `db:"uuid" json:"uuid"` + SpID int64 `db:"sp_id" json:"sp_id"` + Started bool `db:"started" json:"started"` + PieceCid string `db:"piece_cid" json:"piece_cid"` + PieceSize int64 `db:"piece_size" json:"piece_size"` + PieceCidV2 string `db:"-" json:"piece_cid_v2"` + RawSize sql.NullInt64 `db:"raw_size" json:"raw_size"` + Offline bool `db:"offline" json:"offline"` + URL *string `db:"url" json:"url"` + Headers []byte `db:"headers" json:"headers"` + CommTaskID *int64 `db:"commp_task_id" json:"commp_task_id"` + AfterCommp bool `db:"after_commp" json:"after_commp"` + PSDTaskID *int64 `db:"psd_task_id" json:"psd_task_id"` + AfterPSD bool `db:"after_psd" json:"after_psd"` + PSDWaitTime *time.Time `db:"psd_wait_time" json:"psd_wait_time"` + FindDealTaskID *int64 `db:"find_deal_task_id" json:"find_deal_task_id"` + AfterFindDeal bool `db:"after_find_deal" json:"after_find_deal"` + Sector *int64 `db:"sector" json:"sector"` + Offset *int64 `db:"sector_offset" json:"sector_offset"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + Indexed bool `db:"indexed" json:"indexed"` + Announce bool `db:"announce" json:"announce"` + Complete bool `db:"complete" json:"complete"` + Miner string `json:"miner"` } func (a *WebRPC) GetMK12DealPipelines(ctx context.Context, limit int, offset int) ([]*MK12Pipeline, error) { @@ -168,18 +168,17 @@ func (a *WebRPC) GetMK12DealPipelines(ctx context.Context, limit int, offset int return nil, xerrors.Errorf("failed to parse the miner ID: %w", err) } s.Miner = addr.String() - pcid, err := cid.Parse(s.PieceCid) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(s.PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + if s.RawSize.Valid { + pcid, err := cid.Parse(s.PieceCid) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(s.RawSize.Int64)) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + s.PieceCidV2 = pcid2.String() } - s.PieceCidV2 = commp.PCidV2().String() } return pipelines, nil @@ -188,7 +187,7 @@ func (a *WebRPC) GetMK12DealPipelines(ctx context.Context, limit int, offset int type StorageDealSummary struct { ID string `db:"uuid" json:"id"` MinerID int64 `db:"sp_id" json:"sp_id"` - Sector *int64 `db:"sector_num" json:"sector"` + Sector sql.NullInt64 `db:"sector_num" json:"sector"` CreatedAt time.Time `db:"created_at" json:"created_at"` SignedProposalCid string `db:"signed_proposal_cid" json:"signed_proposal_cid"` Offline bool `db:"offline" json:"offline"` @@ -196,10 +195,11 @@ type StorageDealSummary struct { StartEpoch int64 `db:"start_epoch" json:"start_epoch"` EndEpoch int64 `db:"end_epoch" json:"end_epoch"` ClientPeerId string `db:"client_peer_id" json:"client_peer_id"` - ChainDealId *int64 `db:"chain_deal_id" json:"chain_deal_id"` - PublishCid *string `db:"publish_cid" json:"publish_cid"` + ChainDealId sql.NullInt64 `db:"chain_deal_id" json:"chain_deal_id"` + PublishCid sql.NullString `db:"publish_cid" json:"publish_cid"` PieceCid string `db:"piece_cid" json:"piece_cid"` PieceSize int64 `db:"piece_size" json:"piece_size"` + RawSize sql.NullInt64 `db:"raw_size"` FastRetrieval bool `db:"fast_retrieval" json:"fast_retrieval"` AnnounceToIpni bool `db:"announce_to_ipni" json:"announce_to_ipni"` Url sql.NullString `db:"url"` @@ -209,29 +209,18 @@ type StorageDealSummary struct { DBError sql.NullString `db:"error"` Error string `json:"error"` Miner string `json:"miner"` - IsLegacy bool `json:"is_legacy"` - Indexed *bool `db:"indexed" json:"indexed"` + Indexed sql.NullBool `db:"indexed" json:"indexed"` IsDDO bool `db:"is_ddo" json:"is_ddo"` + PieceCidV2 string `json:"piece_cid_v2"` } func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDealSummary, error) { - - var isLegacy bool - var pcid cid.Cid - id, err := uuid.Parse(deal) if err != nil { - p, perr := cid.Parse(deal) - if perr != nil { - return &StorageDealSummary{}, xerrors.Errorf("failed to parse the deal ID: %w and %w", err, perr) - } - isLegacy = true - pcid = p + return nil, xerrors.Errorf("failed to parse deal ID: %w", err) } - - if !isLegacy { - var summaries []StorageDealSummary - err = a.deps.DB.Select(ctx, &summaries, `SELECT + var summaries []StorageDealSummary + err = a.deps.DB.Select(ctx, &summaries, `SELECT deal.uuid, deal.sp_id, deal.created_at, @@ -245,6 +234,7 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal deal.publish_cid, deal.piece_cid, deal.piece_size, + deal.raw_size, deal.fast_retrieval, deal.announce_to_ipni, deal.url, @@ -269,6 +259,7 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal md.publish_cid, md.piece_cid, md.piece_size, + md.raw_size, md.fast_retrieval, md.announce_to_ipni, md.url, @@ -295,6 +286,7 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal '' AS publish_cid, mdd.piece_cid, mdd.piece_size, + mdd.raw_size, mdd.fast_retrieval, mdd.announce_to_ipni, '' AS url, @@ -307,95 +299,61 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal LEFT JOIN market_piece_deal mpd ON mpd.id = deal.uuid AND mpd.sp_id = deal.sp_id LEFT JOIN market_piece_metadata mpm - ON mpm.piece_cid = deal.piece_cid; + ON mpm.piece_cid = deal.piece_cid AND mpm.piece_size = deal.piece_size; `, id.String()) - if err != nil { - return &StorageDealSummary{}, xerrors.Errorf("select deal summary: %w", err) - } - - if len(summaries) == 0 { - return nil, xerrors.Errorf("No such deal found in database: %s", id.String()) - } - - d := summaries[0] - d.IsLegacy = isLegacy - - addr, err := address.NewIDAddress(uint64(d.MinerID)) - if err != nil { - return &StorageDealSummary{}, err - } - - if d.Header != nil { - var h http.Header - err = json.Unmarshal(d.Header, &h) - if err != nil { - return &StorageDealSummary{}, err - } - d.UrlHeaders = h - } - - if !d.Url.Valid { - d.URLS = "" - } else { - d.URLS = d.Url.String - } - - if !d.DBError.Valid { - d.Error = "" - } else { - d.Error = d.DBError.String - } - - d.Miner = addr.String() - - return &d, nil - } - - var summaries []StorageDealSummary - err = a.deps.DB.Select(ctx, &summaries, `SELECT - '' AS uuid, - sp_id, - created_at, - signed_proposal_cid, - FALSE as offline, - verified, - start_epoch, - end_epoch, - client_peer_id, - chain_deal_id, - publish_cid, - piece_cid, - piece_size, - fast_retrieval, - FALSE AS announce_to_ipni, - '' AS url, - '{}' AS url_headers, - '' AS error, - sector_num, - FALSE AS indexed - FROM market_legacy_deals - WHERE signed_proposal_cid = $1`, pcid.String()) - if err != nil { - return &StorageDealSummary{}, err + return &StorageDealSummary{}, xerrors.Errorf("select deal summary: %w", err) } if len(summaries) == 0 { - return nil, xerrors.Errorf("No such deal found in database :%s", pcid.String()) + return nil, xerrors.Errorf("No such deal found in database: %s", id.String()) } d := summaries[0] - d.IsLegacy = isLegacy addr, err := address.NewIDAddress(uint64(d.MinerID)) if err != nil { return &StorageDealSummary{}, err } + if d.Header != nil { + var h http.Header + err = json.Unmarshal(d.Header, &h) + if err != nil { + return &StorageDealSummary{}, err + } + d.UrlHeaders = h + } + + if !d.Url.Valid { + d.URLS = "" + } else { + d.URLS = d.Url.String + } + + if !d.DBError.Valid { + d.Error = "" + } else { + d.Error = d.DBError.String + } + d.Miner = addr.String() + if d.RawSize.Valid { + pcid, err := cid.Parse(d.PieceCid) + if err != nil { + return &StorageDealSummary{}, xerrors.Errorf("failed to parse piece CID: %w", err) + } + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(d.RawSize.Int64)) + if err != nil { + return &StorageDealSummary{}, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + d.PieceCidV2 = pcid2.String() + } + return &d, nil + } type StorageDealList struct { @@ -404,6 +362,7 @@ type StorageDealList struct { CreatedAt time.Time `db:"created_at" json:"created_at"` PieceCidV1 string `db:"piece_cid" json:"piece_cid"` PieceSize int64 `db:"piece_size" json:"piece_size"` + RawSize sql.NullInt64 `db:"raw_size"` PieceCidV2 string `json:"piece_cid_v2"` Processed bool `db:"processed" json:"processed"` Error sql.NullString `db:"error" json:"error"` @@ -419,6 +378,7 @@ func (a *WebRPC) MK12StorageDealList(ctx context.Context, limit int, offset int) md.created_at, md.piece_cid, md.piece_size, + md.raw_size, md.error, coalesce(mm12dp.complete, true) as processed FROM market_mk12_deals md @@ -435,62 +395,65 @@ func (a *WebRPC) MK12StorageDealList(ctx context.Context, limit int, offset int) return nil, err } mk12Summaries[i].Miner = addr.String() - pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + + // Find PieceCidV2 only of rawSize is present + // It will be absent only for Offline deals (mk12, mk12-ddo), waiting for data + if mk12Summaries[i].RawSize.Valid { + pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(mk12Summaries[i].RawSize.Int64)) + if err != nil { + return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + } + mk12Summaries[i].PieceCidV2 = pcid2.String() } - mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() } return mk12Summaries, nil } -func (a *WebRPC) LegacyStorageDealList(ctx context.Context, limit int, offset int) ([]StorageDealList, error) { - var mk12Summaries []StorageDealList - - err := a.deps.DB.Select(ctx, &mk12Summaries, `SELECT - signed_proposal_cid AS uuid, - sp_id, - created_at, - piece_cid, - piece_size, - NULL AS error, - TRUE AS processed - FROM market_legacy_deals - ORDER BY created_at DESC - LIMIT $1 OFFSET $2;`, limit, offset) - if err != nil { - return nil, fmt.Errorf("failed to fetch deal list: %w", err) - } - - for i := range mk12Summaries { - addr, err := address.NewIDAddress(uint64(mk12Summaries[i].MinerID)) - if err != nil { - return nil, err - } - mk12Summaries[i].Miner = addr.String() - pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) - } - mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() - } - return mk12Summaries, nil -} +// LegacyStorageDealList is deprecated +//func (a *WebRPC) LegacyStorageDealList(ctx context.Context, limit int, offset int) ([]StorageDealList, error) { +// var mk12Summaries []StorageDealList +// +// err := a.deps.DB.Select(ctx, &mk12Summaries, `SELECT +// signed_proposal_cid AS uuid, +// sp_id, +// created_at, +// piece_cid, +// piece_size, +// NULL AS error, +// TRUE AS processed +// FROM market_legacy_deals +// ORDER BY created_at DESC +// LIMIT $1 OFFSET $2;`, limit, offset) +// if err != nil { +// return nil, fmt.Errorf("failed to fetch deal list: %w", err) +// } +// +// for i := range mk12Summaries { +// addr, err := address.NewIDAddress(uint64(mk12Summaries[i].MinerID)) +// if err != nil { +// return nil, err +// } +// mk12Summaries[i].Miner = addr.String() +// pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) +// if err != nil { +// return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) +// } +// commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ +// PieceCID: pcid, +// Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), +// }) +// if err != nil { +// return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) +// } +// mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() +// } +// return mk12Summaries, nil +//} type WalletBalances struct { Address string `json:"address"` @@ -643,6 +606,10 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er return nil, err } + if !commcidv2.IsPieceCidV2(piece) { + return nil, xerrors.Errorf("invalid piece CID V2: %w", err) + } + commp, err := commcidv2.CommPFromPCidV2(piece) if err != nil { return nil, xerrors.Errorf("failed to get commP from piece CID: %w", err) @@ -652,6 +619,8 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er ret := &PieceInfo{ PieceCidv2: piece.String(), + PieceCid: pi.PieceCID.String(), + Size: int64(pi.Size), } err = a.deps.DB.QueryRow(ctx, `SELECT created_at, indexed, indexed_at FROM market_piece_metadata WHERE piece_cid = $1 AND piece_size = $2`, pi.PieceCID.String(), pi.Size).Scan(&ret.CreatedAt, &ret.Indexed, &ret.IndexedAT) @@ -687,10 +656,8 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er pieceDeals[i].MK20 = true } pieceDeals[i].Miner = addr.String() - ret.Size = pieceDeals[i].Length } ret.Deals = pieceDeals - ret.PieceCid = pi.PieceCID.String() b := new(bytes.Buffer) @@ -736,6 +703,10 @@ func (a *WebRPC) PieceParkStates(ctx context.Context, pieceCID string) (*ParkedP return nil, err } + if !commcidv2.IsPieceCidV2(pcid) { + return nil, xerrors.Errorf("invalid piece CID V2: %w", err) + } + commp, err := commcidv2.CommPFromPCidV2(pcid) if err != nil { return nil, xerrors.Errorf("failed to get commP from piece CID: %w", err) @@ -868,9 +839,10 @@ type MK20DealPipeline struct { SpId int64 `db:"sp_id" json:"sp_id"` Contract string `db:"contract" json:"contract"` Client string `db:"client" json:"client"` + PieceCidV2 string `db:"piece_cid_v2" json:"piece_cid_v2"` PieceCid string `db:"piece_cid" json:"piece_cid"` PieceSize int64 `db:"piece_size" json:"piece_size"` - RawSize sql.NullInt64 `db:"raw_size" json:"raw_size"` + RawSize uint64 `db:"raw_size" json:"raw_size"` Offline bool `db:"offline" json:"offline"` URL sql.NullString `db:"url" json:"url"` Indexing bool `db:"indexing" json:"indexing"` @@ -902,8 +874,7 @@ type MK20DealPipeline struct { Complete bool `db:"complete" json:"complete"` CreatedAt time.Time `db:"created_at" json:"created_at"` - Miner string `db:"-" json:"miner"` - PieceCidV2 string `db:"-" json:"piece_cid_v2"` + Miner string `db:"-" json:"miner"` } type PieceInfoMK12Deals struct { @@ -928,6 +899,10 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe return nil, err } + if !commcidv2.IsPieceCidV2(pcid) { + return nil, xerrors.Errorf("invalid piece CID V2: %w", err) + } + commp, err := commcidv2.CommPFromPCidV2(pcid) if err != nil { return nil, err @@ -1053,15 +1028,11 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe var mk20Deals []*mk20.DBDeal err = a.deps.DB.Select(ctx, &mk20Deals, `SELECT id, - piece_cid, - piece_size, - format, - source_http, - source_aggregate, - source_offline, - source_http_put, + client, + data, ddo_v1, - error FROM market_mk20_deal WHERE piece_cid = $1 AND piece_size = $2`, pieceCid, size) + retrieval_v1, + pdp_v1 FROM market_mk20_deal WHERE piece_cid_v2 = $1`, pcid.String()) if err != nil { return nil, xerrors.Errorf("failed to query mk20 deals: %w", err) } @@ -1075,9 +1046,20 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe return nil, err } ids[i] = deal.Identifier.String() + + var Err sql.NullString + + if len(dbdeal.DDOv1) > 0 && string(dbdeal.DDOv1) != "null" { + var dddov1 mk20.DBDDOV1 + if err := json.Unmarshal(dbdeal.DDOv1, &dddov1); err != nil { + return nil, fmt.Errorf("unmarshal ddov1: %w", err) + } + Err = dddov1.Error + } + mk20deals[i] = &MK20StorageDeal{ Deal: deal, - Error: dbdeal.Error, + Error: Err, } } @@ -1089,6 +1071,7 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe sp_id, contract, client, + piece_cid_v2, piece_cid, piece_size, raw_size, @@ -1779,6 +1762,7 @@ func (a *WebRPC) MK12DDOStorageDealList(ctx context.Context, limit int, offset i md.created_at, md.piece_cid, md.piece_size, + md.raw_size, md.error, coalesce(mm12dp.complete, true) as processed FROM market_direct_deals md @@ -1795,18 +1779,18 @@ func (a *WebRPC) MK12DDOStorageDealList(ctx context.Context, limit int, offset i return nil, err } mk12Summaries[i].Miner = addr.String() - pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) + + if mk12Summaries[i].RawSize.Valid { + pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) + if err != nil { + return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) + } + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(mk12Summaries[i].RawSize.Int64)) + if err != nil { + return nil, xerrors.Errorf("failed to convert v1 piece CID to v2: %w", err) + } + mk12Summaries[i].PieceCidV2 = pcid2.String() } - mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() } return mk12Summaries, nil diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index b607b8e54..fe71f21b8 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -3,6 +3,7 @@ package webrpc import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "strconv" @@ -11,23 +12,19 @@ import ( eabi "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" - "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" - "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/mk20" ) type MK20StorageDeal struct { - Deal *mk20.Deal `json:"deal"` - Error sql.NullString `json:"error"` - PieceCidV2 string `json:"piece_cid_v2"` + Deal *mk20.Deal `json:"deal"` + Error sql.NullString `json:"error"` } func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20StorageDeal, error) { @@ -36,51 +33,58 @@ func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20Storag return nil, xerrors.Errorf("parsing deal ID: %w", err) } - var dbDeal []mk20.DBDeal - err = a.deps.DB.Select(ctx, &dbDeal, `SELECT id, - piece_cid, - piece_size, - format, - source_http, - source_aggregate, - source_offline, - source_http_put, + var dbDeals []mk20.DBDeal + err = a.deps.DB.Select(ctx, &dbDeals, `SELECT id, + client, + data, ddo_v1, - error FROM market_mk20_deal WHERE id = $1`, pid.String()) + retrieval_v1, + pdp_v1 FROM market_mk20_deal WHERE id = $1`, pid.String()) if err != nil { return nil, xerrors.Errorf("getting deal from DB: %w", err) } - if len(dbDeal) != 1 { - return nil, xerrors.Errorf("expected 1 deal, got %d", len(dbDeal)) + if len(dbDeals) != 1 { + return nil, xerrors.Errorf("expected 1 deal, got %d", len(dbDeals)) } - deal, err := dbDeal[0].ToDeal() + dbDeal := dbDeals[0] + deal, err := dbDeal.ToDeal() if err != nil { return nil, xerrors.Errorf("converting DB deal to struct: %w", err) } - pi := abi.PieceInfo{ - PieceCID: deal.Data.PieceCID, - Size: deal.Data.Size, - } + ret := &MK20StorageDeal{Deal: deal} - commp, err := commcidv2.CommPFromPieceInfo(pi) - if err != nil { - return nil, xerrors.Errorf("failed to get commp: %w", err) + if len(dbDeal.DDOv1) > 0 && string(dbDeal.DDOv1) != "null" { + var dddov1 mk20.DBDDOV1 + if err := json.Unmarshal(dbDeal.DDOv1, &dddov1); err != nil { + return nil, fmt.Errorf("unmarshal ddov1: %w", err) + } + if dddov1.Error.Valid { + ret.Error = dddov1.Error + } } - return &MK20StorageDeal{Deal: deal, Error: dbDeal[0].Error, PieceCidV2: commp.PCidV2().String()}, nil + return ret, nil } -func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) ([]*StorageDealList, error) { - var mk20Summaries []*StorageDealList +type MK20StorageDealList struct { + ID string `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + PieceCidV2 sql.NullString `db:"piece_cid_v2" json:"piece_cid_v2"` + Processed bool `db:"processed" json:"processed"` + Error sql.NullString `db:"error" json:"error"` + Miner sql.NullString `db:"miner" json:"miner"` +} + +func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) ([]*MK20StorageDealList, error) { + var mk20Summaries []*MK20StorageDealList err := a.deps.DB.Select(ctx, &mk20Summaries, `SELECT - d.id AS uuid, - d.piece_cid, - d.piece_size, - d.created_at, - d.sp_id, - d.error, + d.created_at, + d.id, + d.piece_cid_v2, + d.ddo_v1->'ddo'->>'provider' AS miner, + d.ddo_v1->>'error' AS error, CASE WHEN EXISTS ( SELECT 1 FROM market_mk20_pipeline_waiting w @@ -95,31 +99,11 @@ func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) FROM market_mk20_deal d WHERE d.ddo_v1 IS NOT NULL AND d.ddo_v1 != 'null' ORDER BY d.created_at DESC - LIMIT $1 OFFSET $2; - `, limit, offset) + LIMIT $1 OFFSET $2;`, limit, offset) if err != nil { return nil, fmt.Errorf("failed to fetch deal list: %w", err) } - for i := range mk20Summaries { - addr, err := address.NewIDAddress(uint64(mk20Summaries[i].MinerID)) - if err != nil { - return nil, err - } - mk20Summaries[i].Miner = addr.String() - pcid, err := cid.Parse(mk20Summaries[i].PieceCidV1) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(mk20Summaries[i].PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) - } - mk20Summaries[i].PieceCidV2 = commp.PCidV2().String() - } return mk20Summaries, nil } @@ -142,6 +126,7 @@ func (a *WebRPC) MK20DealPipelines(ctx context.Context, limit int, offset int) ( sp_id, contract, client, + piece_cid_v2, piece_cid, piece_size, raw_size, @@ -180,18 +165,6 @@ func (a *WebRPC) MK20DealPipelines(ctx context.Context, limit int, offset int) ( return nil, xerrors.Errorf("failed to parse the miner ID: %w", err) } s.Miner = addr.String() - pcid, err := cid.Parse(s.PieceCid) - if err != nil { - return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) - } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(s.PieceSize), - }) - if err != nil { - return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) - } - s.PieceCidV2 = commp.PCidV2().String() } return pipelines, nil diff --git a/web/api/webrpc/pdp.go b/web/api/webrpc/pdp.go index c6c2f99a1..697b94158 100644 --- a/web/api/webrpc/pdp.go +++ b/web/api/webrpc/pdp.go @@ -11,7 +11,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/yugabyte/pgx/v5" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" ) diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index 6829bf569..273bb6470 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -521,18 +521,15 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto return nil, xerrors.Errorf("failed to parse piece cid: %w", err) } - pi := abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(uint64(pieces[i].PieceSize)), - } + if pieces[i].DataRawSize != nil { + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(*pieces[i].DataRawSize)) + if err != nil { + return nil, xerrors.Errorf("failed to generate piece cid v2: %w", err) + } - commp, err := commcidv2.CommPFromPieceInfo(pi) - if err != nil { - return nil, xerrors.Errorf("failed to parse piece cid: %w", err) + pieces[i].PieceCidV2 = pcid2.String() } - pieces[i].PieceCidV2 = commp.PCidV2().String() - id, isPiecePark := strings.CutPrefix(derefOrZero(pieces[i].DataUrl), "pieceref:") if !isPiecePark { continue diff --git a/web/static/pages/market-settings/allow-list.mjs b/web/static/pages/market-settings/allow-list.mjs index 08a1b9c13..4edc87cd3 100644 --- a/web/static/pages/market-settings/allow-list.mjs +++ b/web/static/pages/market-settings/allow-list.mjs @@ -25,7 +25,6 @@ class AllowList extends LitElement { if (Array.isArray(result)) { this.allowList = result; } else { - console.error('GetAllowDenyList did not return an array:', result); this.allowList = []; } } catch (error) { diff --git a/web/static/pages/mk12-deal/deal.mjs b/web/static/pages/mk12-deal/deal.mjs index 1c06cce8d..3ad8a6e48 100644 --- a/web/static/pages/mk12-deal/deal.mjs +++ b/web/static/pages/mk12-deal/deal.mjs @@ -33,17 +33,21 @@ class DealDetails extends LitElement { {property: 'Signed Proposal Cid', value: entry.signed_proposal_cid}, {property: 'Offline', value: entry.offline}, {property: 'Verified', value: entry.verified}, - {property: 'Is Legacy', value: entry.is_legacy}, {property: 'Is DDO', value: entry.is_ddo}, {property: 'Start Epoch', value: html``}, {property: 'End Epoch', value: html``}, {property: 'Client Peer ID', value: html``}, {property: 'Chain Deal ID', value: entry.chain_deal_id}, {property: 'Publish CID', value: entry.publish_cid}, - {property: 'Piece CID', value: html`${entry.piece_cid}`}, + {property: 'Piece CID', value: entry.piece_cid}, + {property: 'Piece CID V2', value: entry.piece_cid_v2 && entry.piece_cid_v2.trim() !== '' + ? html`${entry.piece_cid_v2}` + : 'N/A'}, {property: 'Piece Size', value: entry.piece_size}, + {property: 'Raw Size', value: entry.raw_size || 'N/A'}, {property: 'Fast Retrieval', value: entry.fast_retrieval}, {property: 'Announce To IPNI', value: entry.announce_to_ipni}, + {property: 'Indexed', value: entry.indexed ? 'Yes' : 'No'}, {property: 'Url', value: entry.url}, {property: 'Url Headers', value: html`
    diff --git a/web/static/pages/mk12-deals/deal-pipelines.mjs b/web/static/pages/mk12-deals/deal-pipelines.mjs index 9e73ac5b2..9f1fc392e 100644 --- a/web/static/pages/mk12-deals/deal-pipelines.mjs +++ b/web/static/pages/mk12-deals/deal-pipelines.mjs @@ -41,7 +41,7 @@ class DealPipelines extends LitElement { try { const params = [this.limit, this.offset]; const deals = await RPCCall('GetMK12DealPipelines', params); - this.deals = deals; + this.deals = deals || []; // Load failed tasks data const failed = await RPCCall('MK12PipelineFailedTasks', []); @@ -224,7 +224,10 @@ class DealPipelines extends LitElement { ${deal.miner} - ${this.formatPieceCid(deal.piece_cid)} + ${deal.piece_cid_v2 && deal.piece_cid_v2 !== "" + ? html`${this.formatPieceCid(deal.piece_cid_v2)}` + : html`${this.formatPieceCid(deal.piece_cid)}` + } ${this.formatBytes(deal.piece_size)} ${this.getDealStatus(deal)} diff --git a/web/static/pages/mk12-deals/index.html b/web/static/pages/mk12-deals/index.html index 0964ecce0..d4c3c1f27 100644 --- a/web/static/pages/mk12-deals/index.html +++ b/web/static/pages/mk12-deals/index.html @@ -8,7 +8,6 @@ - @@ -52,13 +51,6 @@

    Storage Deals

    -
    -
    -
    - -
    -
    -
    diff --git a/web/static/pages/mk12-deals/mk12-deals.mjs b/web/static/pages/mk12-deals/mk12-deals.mjs index 056cf0be1..bcf0bf1ee 100644 --- a/web/static/pages/mk12-deals/mk12-deals.mjs +++ b/web/static/pages/mk12-deals/mk12-deals.mjs @@ -89,13 +89,18 @@ class MK12DealList extends LitElement { ${this.deals.map( (deal) => html` - ${formatDate(deal.created_at)} - ${deal.id} - ${deal.miner} - ${this.formatPieceCid(deal.piece_cid)} - ${this.formatBytes(deal.piece_size)} - - + ${formatDate(deal.created_at)} + ${deal.id} + ${deal.miner} + + ${deal.piece_cid_v2 && deal.piece_cid_v2 !== "" + ? html`${this.formatPieceCid(deal.piece_cid_v2)}` + : html`${this.formatPieceCid(deal.piece_cid)}` + } + + ${this.formatBytes(deal.piece_size)} + + ` )} diff --git a/web/static/pages/mk12-deals/mk12ddo-list.mjs b/web/static/pages/mk12-deals/mk12ddo-list.mjs index 3fb34796f..609c8e4a9 100644 --- a/web/static/pages/mk12-deals/mk12ddo-list.mjs +++ b/web/static/pages/mk12-deals/mk12ddo-list.mjs @@ -88,13 +88,18 @@ class MK12DDODealList extends LitElement { ${this.deals.map( (deal) => html` - ${formatDate(deal.created_at)} - ${deal.id} - ${deal.miner} - ${deal.piece_cid} - ${this.formatBytes(deal.piece_size)} - - + ${formatDate(deal.created_at)} + ${deal.id} + ${deal.miner} + + ${deal.piece_cid_v2 && deal.piece_cid_v2 !== "" + ? html`${this.formatPieceCid(deal.piece_cid_v2)}` + : html`${this.formatPieceCid(deal.piece_cid)}` + } + + ${this.formatBytes(deal.piece_size)} + + ` )} @@ -132,6 +137,16 @@ class MK12DDODealList extends LitElement { } } + formatPieceCid(pieceCid) { + if (!pieceCid) return ''; + if (pieceCid.length <= 24) { + return pieceCid; + } + const start = pieceCid.substring(0, 16); + const end = pieceCid.substring(pieceCid.length - 8); + return `${start}...${end}`; + } + static styles = css` .pagination-controls { display: flex; diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 5523f9e5b..3eac0270e 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -30,7 +30,7 @@ class DealDetails extends LitElement { render() { if (!this.data) return html`

    No data.

    `; - const { identifier, data, products, error } = this.data.deal; + const { identifier, client, data, products, error } = this.data.deal; return html` @@ -43,9 +43,16 @@ class DealDetails extends LitElement { + - - + + + +
    Identifier${identifier}
    Client
    Error
    PieceCID${data?.piece_cid['/']}
    PieceSize${data?.piece_size}
    PieceCID + ${data + ? html`${data.piece_cid['/']}` + : "Not Available"} +

    Piece Format

    @@ -63,6 +70,7 @@ class DealDetails extends LitElement { ${products?.ddo_v1 ? this.renderDDOV1(products.ddo_v1) : ''} + ${products?.retrieval_v1 ? this.renderRetV1(products.retrieval_v1) : ''} `; } @@ -176,7 +184,7 @@ class DealDetails extends LitElement {
      -
    • PieceCID: ${piece.piece_cid['/']} Size: ${piece.piece_size}
      • +
    • PieceCID: ${piece.piece_cid['/']}
    • ${this.renderPieceFormat(piece.format)}
    • ${this.renderDataSource(piece)}
    @@ -206,19 +214,28 @@ class DealDetails extends LitElement { } renderDDOV1(ddo) { + if (!ddo) return ''; return html`
    DDO v1
    - ${ddo.allocation_id ? html`` : ''} - - +
    Provider${ddo.provider}
    Client
    Piece Manager
    Duration${ddo.duration}
    Allocation ID${ddo.allocation_id}
    Contract${ddo.contract_address}
    Verify Method${ddo.contract_verify_method}
    Notify Address${ddo.notification_address}
    Indexing${ddo.indexing ? 'Yes' : 'No'}
    Announce to IPNI${ddo.announce_to_ipni ? 'Yes' : 'No'}
    + `; + } + + renderRetV1(ret) { + if (!ret) return ''; + return html` +
    Retrieval v1
    + + +
    Indexing${ret.indexing ? 'Yes' : 'No'}
    Announce to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    `; } diff --git a/web/static/pages/mk20/ddo-pipeline.mjs b/web/static/pages/mk20/ddo-pipeline.mjs index e422ebb90..001751ba7 100644 --- a/web/static/pages/mk20/ddo-pipeline.mjs +++ b/web/static/pages/mk20/ddo-pipeline.mjs @@ -41,7 +41,7 @@ class MK20DealPipelines extends LitElement { try { const params = [this.limit, this.offset]; const deals = await RPCCall('MK20DealPipelines', params); - this.deals = deals; + this.deals = deals || []; // Load failed tasks data const failed = await RPCCall('MK20PipelineFailedTasks', []); @@ -207,7 +207,6 @@ class MK20DealPipelines extends LitElement { UUID SP ID Piece CID - Piece Size Status @@ -221,9 +220,8 @@ class MK20DealPipelines extends LitElement { ${deal.miner} - ${this.formatPieceCid(deal.piece_cid)} + ${this.formatPieceCid(deal.piece_cid_v2)} - ${this.formatBytes(deal.piece_size)} ${this.getDealStatus(deal)} ` diff --git a/web/static/pages/mk20/ddo.mjs b/web/static/pages/mk20/ddo.mjs index c6c4b39b2..4dd275294 100644 --- a/web/static/pages/mk20/ddo.mjs +++ b/web/static/pages/mk20/ddo.mjs @@ -89,13 +89,16 @@ class MK20DDODealList extends LitElement { ${this.deals.map( (deal) => html` - ${formatDate(deal.created_at)} - ${deal.id} - ${deal.miner} - ${this.formatPieceCid(deal.piece_cid)} - ${this.formatBytes(deal.piece_size)} - - + ${formatDate(deal.created_at)} + ${deal.id} + ${deal.miner.Valid ? deal.miner.String : '-'} + + ${deal.piece_cid_v2 + ? html`${this.formatPieceCid(deal.piece_cid_v2.String)}` + : 'Not Available'} + + + ` )} diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index b522f0e93..4d7817280 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -307,8 +307,8 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement {
    Sealing šŸ“¦
    Sector${entry.mk12_pipeline.sector.Valid ? html`${entry.mk12_pipeline.sector.Int64}` : 'N/A'} - Reg Seal Proof${entry.mk12_pipeline.reg_seal_proof.Valid ? entry.pipeline.reg_seal_proof.Int64 : 'N/A'} - Sector Offset${entry.mk12_pipeline.sector_offset.Valid ? entry.pipeline.sector_offset.Int64 : 'N/A'} + Reg Seal Proof${entry.mk12_pipeline.reg_seal_proof.Valid ? entry.mk12_pipeline.reg_seal_proof.Int64 : 'N/A'} + Sector Offset${entry.mk12_pipeline.sector_offset.Valid ? entry.mk12_pipeline.sector_offset.Int64 : 'N/A'}
    Indexing šŸ”
    Indexing Created At${entry.mk12_pipeline.indexing_created_at.Valid ? formatDate(entry.mk12_pipeline.indexing_created_at.Time) : 'N/A'} diff --git a/web/static/pages/sector/sector-info.mjs b/web/static/pages/sector/sector-info.mjs index ca1cbdab9..023cb4911 100644 --- a/web/static/pages/sector/sector-info.mjs +++ b/web/static/pages/sector/sector-info.mjs @@ -111,6 +111,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement { + @@ -130,7 +131,12 @@ customElements.define('sector-info',class SectorInfo extends LitElement { ${(this.data.Pieces||[]).map(piece => html` - + + From d7ba6f380f8d4e54ea174f6a8dbbbad7b2643be9 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 8 Jul 2025 14:48:07 +0400 Subject: [PATCH 18/55] PDP pipeline --- cmd/curio/tasks/tasks.go | 7 +- .../harmonydb/sql/20250505-market_mk20.sql | 113 +++- lib/cachedreader/cachedreader.go | 47 +- market/indexstore/cql/0002_piece_index.cql | 7 + market/indexstore/indexstore.go | 123 ++++ market/mk20/http/http.go | 77 ++- market/mk20/http/info.md | 8 +- market/mk20/mk20.go | 382 ++++++++++- market/mk20/mk20_upload.go | 116 ++-- market/mk20/pdp_v1.go | 106 ++- market/mk20/utils.go | 95 ++- pdp/contract/types.go | 9 + pdp/handlers.go | 2 +- tasks/pdp/proofset_addroot_watch.go | 175 +++-- tasks/pdp/proofset_create_watch.go | 100 ++- tasks/pdp/task_add_proofset.go | 183 ++++++ tasks/pdp/task_addroot.go | 450 +++++++------ tasks/pdp/task_aggregation.go | 353 ++++++++++ tasks/pdp/task_init_pp.go | 12 +- tasks/pdp/task_next_pp.go | 8 +- tasks/pdp/task_prove.go | 396 ++++++------ tasks/pdp/task_save_cache.go | 604 ++++++++++++++++++ tasks/piece/task_aggregate_chunks.go | 71 +- tasks/storage-market/mk20.go | 44 +- tasks/storage-market/task_aggregation.go | 18 +- 25 files changed, 2838 insertions(+), 668 deletions(-) create mode 100644 pdp/contract/types.go create mode 100644 tasks/pdp/task_add_proofset.go create mode 100644 tasks/pdp/task_aggregation.go create mode 100644 tasks/pdp/task_save_cache.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index fb56a31e7..23786c6bf 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -289,11 +289,14 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan pdp.NewWatcherCreate(db, must.One(dependencies.EthClient.Val()), chainSched) pdp.NewWatcherRootAdd(db, must.One(dependencies.EthClient.Val()), chainSched) - pdpProveTask := pdp.NewProveTask(chainSched, db, must.One(dependencies.EthClient.Val()), dependencies.Chain, es, dependencies.CachedPieceReader) + pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) + pdpCache := pdp.NewTaskSavePDPCache(db, dependencies.CachedPieceReader, iStore) + pdpAddRoot := pdp.NewPDPTaskAddRoot(db, es, must.One(dependencies.EthClient.Val())) + pdpProveTask := pdp.NewProveTask(chainSched, db, must.One(dependencies.EthClient.Val()), dependencies.Chain, es, dependencies.CachedPieceReader, iStore) pdpNextProvingPeriodTask := pdp.NewNextProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpInitProvingPeriodTask := pdp.NewInitProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpNotifTask := pdp.NewPDPNotifyTask(db) - activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask) + activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, pdpAddRoot, pdpAggregateTask, pdpCache) } idxMax := taskhelp.Max(cfg.Subsystems.IndexingMaxTasks) diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 3a5c50861..8dc8d6d44 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -226,10 +226,11 @@ CREATE TABLE market_mk20_pipeline_waiting ( CREATE TABLE market_mk20_download_pipeline ( id TEXT NOT NULL, + product TEXT NOT NULL, -- This allows us to run multiple refs per product for easier lifecycle management piece_cid TEXT NOT NULL, -- This is pieceCid V1 to allow easy table lookups piece_size BIGINT NOT NULL, ref_ids BIGINT[] NOT NULL, - PRIMARY KEY (id, piece_cid, piece_size) + PRIMARY KEY (id, product, piece_cid, piece_size) ); CREATE TABLE market_mk20_offline_urls ( @@ -274,7 +275,8 @@ INSERT INTO market_mk20_data_source (name, enabled) VALUES ('put', TRUE); CREATE OR REPLACE FUNCTION process_offline_download( _id TEXT, _piece_cid TEXT, - _piece_size BIGINT + _piece_size BIGINT, + _product TEXT ) RETURNS BOOLEAN AS $$ DECLARE _url TEXT; @@ -320,9 +322,9 @@ BEGIN RETURNING ref_id INTO _ref_id; -- 6. Insert or update download pipeline with ref_id - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - VALUES (_id, _piece_cid, _piece_size, ARRAY[_ref_id]) - ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) + VALUES (_id, _piece_cid, _piece_size, _product, ARRAY[_ref_id]) + ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE SET ref_ids = ( SELECT ARRAY( SELECT DISTINCT r @@ -343,14 +345,113 @@ $$ LANGUAGE plpgsql; ALTER TABLE parked_pieces ADD COLUMN skip BOOLEAN DEFAULT FALSE; +CREATE TABLE pdp_proof_set ( + id BIGINT PRIMARY KEY, -- on-chain proofset id + client TEXT NOT NULL, -- client wallet which requested this proofset + + -- updated when a challenge is requested (either by first proofset add or by invokes of nextProvingPeriod) + -- initially NULL on fresh proofsets. + prev_challenge_request_epoch BIGINT, + + -- task invoking nextProvingPeriod, the task should be spawned any time prove_at_epoch+challenge_window is in the past + challenge_request_task_id BIGINT REFERENCES harmony_task(id) ON DELETE SET NULL, + + -- nextProvingPeriod message hash, when the message lands prove_task_id will be spawned and + -- this value will be set to NULL + challenge_request_msg_hash TEXT, + + -- the proving period for this proofset and the challenge window duration + proving_period BIGINT, + challenge_window BIGINT, + + -- the epoch at which the next challenge window starts and proofs can be submitted + -- initialized to NULL indicating a special proving period init task handles challenge generation + prove_at_epoch BIGINT, + + -- flag indicating that the proving period is ready for init. Currently set after first add + -- Set to true after first root add + init_ready BOOLEAN NOT NULL DEFAULT FALSE, + + create_deal_id TEXT NOT NULL, -- mk20 deal ID for creating this proofset + create_message_hash TEXT NOT NULL, + + remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this proofset + remove_message_hash TEXT DEFAULT NULL, + + unique (create_deal_id), + unique (remove_deal_id) +); + +CREATE TABLE pdp_proof_set_create ( + id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response + client TEXT NOT NULL, + + record_keeper TEXT NOT NULL, + extra_data BYTEA, + task_id BIGINT DEFAULT NULL, + + tx_hash TEXT DEFAULT NULL +); + +CREATE TABLE pdp_proofset_root ( + proofset BIGINT NOT NULL, -- pdp_proof_sets.id + client TEXT NOT NULL, + + piece_cid_v2 TEXT NOT NULL, -- root cid (piececid v2) + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + raw_size BIGINT NOT NULL, + + root BIGINT DEFAULT NULL, -- on-chain index of the root in the rootCids sub-array + + piece_ref BIGINT NOT NULL, -- piece_ref_id + + add_deal_id TEXT NOT NULL, -- mk20 deal ID for adding this root to proofset + add_message_hash TEXT NOT NULL, + add_message_index BIGINT NOT NULL, -- index of root in the add message + + remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this root from proofset + remove_message_hash TEXT DEFAULT NULL, + remove_message_index BIGINT DEFAULT NULL, + + CONSTRAINT pdp_proofset_roots_root_id_unique PRIMARY KEY (proofset, root_id) +); CREATE TABLE pdp_pipeline ( + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + id TEXT PRIMARY KEY, - piece_cid TEXT NOT NULL, -- v2 piece_cid + client TEXT NOT NULL, + piece_cid_v2 TEXT NOT NULL, -- v2 piece_cid + + piece_cid TEXT NOT NULL, + piece_size BIGINT NOT NULL, + raw_size BIGINT NOT NULL, + + proof_set_id BIGINT NOT NULL, + + extra_data BYTEA NOT NULL, + + piece_ref BIGINT DEFAULT NULL, + + downloaded BOOLEAN DEFAULT FALSE, + + deal_aggregation INT NOT NULL DEFAULT 0, + aggr_index BIGINT DEFAULT 0, + agg_task_id BIGINT DEFAULT NULL, + aggregated BOOLEAN DEFAULT FALSE, + + save_cache_task_id BIGINT DEFAULT NULL, + after_save_cache BOOLEAN DEFAULT FALSE, add_root_task_id BIGINT DEFAULT NULL, after_add_root BOOLEAN DEFAULT FALSE, + add_message_hash TEXT NOT NULL, + add_message_index BIGINT NOT NULL DEFAULT 0, -- index of root in the add message + + after_add_root_msg BOOLEAN DEFAULT FALSE, + indexing BOOLEAN DEFAULT FALSE, indexing_created_at TIMESTAMPTZ DEFAULT NULL, indexing_task_id BIGINT DEFAULT NULL, diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index b42976ad4..a476ae417 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -96,10 +96,10 @@ func NewCachedPieceReader(db *harmonydb.DB, sectorReader *pieceprovider.SectorRe } type cachedSectionReader struct { - reader storiface.Reader - cpr *CachedPieceReader - pieceCid cid.Cid - pieceSize abi.UnpaddedPieceSize + reader storiface.Reader + cpr *CachedPieceReader + pieceCid cid.Cid + rawSize uint64 // Signals when the underlying piece reader is ready ready chan struct{} // err is non-nil if there's an error getting the underlying piece reader @@ -131,7 +131,7 @@ func (r *cachedSectionReader) Close() error { return nil } -func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, uint64, error) { // Get all deals containing this piece commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) @@ -143,11 +143,12 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec pieceSize := commp.PieceInfo().Size var deals []struct { - SpID abi.ActorID `db:"sp_id"` - Sector abi.SectorNumber `db:"sector_num"` - Offset abi.PaddedPieceSize `db:"piece_offset"` - Length abi.PaddedPieceSize `db:"piece_length"` - Proof abi.RegisteredSealProof `db:"reg_seal_proof"` + SpID abi.ActorID `db:"sp_id"` + Sector abi.SectorNumber `db:"sector_num"` + Offset abi.PaddedPieceSize `db:"piece_offset"` + Length abi.PaddedPieceSize `db:"piece_length"` + RawSize int64 `db:"raw_size"` + Proof abi.RegisteredSealProof `db:"reg_seal_proof"` } err = cpr.db.Select(ctx, &deals, `SELECT @@ -155,6 +156,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec mpd.sector_num, mpd.piece_offset, mpd.piece_length, + mpd.raw_size, sm.reg_seal_proof FROM market_piece_deal mpd @@ -191,13 +193,13 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec continue } - return reader, dl.Length.Unpadded(), nil + return reader, uint64(dl.RawSize), nil } return nil, 0, merr } -func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, uint64, error) { commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) if err != nil { return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) @@ -235,7 +237,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromPiecePark(ctx context.Context, p return nil, 0, fmt.Errorf("failed to read piece from piece park: %w", err) } - return reader, abi.UnpaddedPieceSize(pieceData[0].PieceRawSize), nil + return reader, uint64(pieceData[0].PieceRawSize), nil } type SubPieceReader struct { @@ -259,7 +261,7 @@ func (s SubPieceReader) ReadAt(p []byte, off int64) (n int, err error) { return s.sr.ReadAt(p, off) } -func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, uint64, error) { pieces, err := cpr.idxStor.FindPieceInAggregate(ctx, pieceCidV2) if err != nil { return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", err) @@ -269,6 +271,11 @@ func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, p return nil, 0, fmt.Errorf("subpiece not found in any aggregate piece") } + pi, err := commcidv2.CommPFromPCidV2(pieceCidV2) + if err != nil { + return nil, 0, xerrors.Errorf("getting piece commitment from piece CID v2: %w", err) + } + var merr error for _, p := range pieces { @@ -282,15 +289,15 @@ func (cpr *CachedPieceReader) getPieceReaderFromAggregate(ctx context.Context, p continue } sr := io.NewSectionReader(reader, int64(p.Offset), int64(p.Size)) - return SubPieceReader{r: reader, sr: sr}, abi.UnpaddedPieceSize(p.Size), nil + return SubPieceReader{r: reader, sr: sr}, pi.PayloadSize(), nil } sr := io.NewSectionReader(reader, int64(p.Offset), int64(p.Size)) - return SubPieceReader{r: reader, sr: sr}, abi.UnpaddedPieceSize(p.Size), nil + return SubPieceReader{r: reader, sr: sr}, pi.PayloadSize(), nil } return nil, 0, fmt.Errorf("failed to find piece in aggregate: %w", merr) } -func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, abi.UnpaddedPieceSize, error) { +func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCidV2 cid.Cid) (storiface.Reader, uint64, error) { cacheKey := pieceCidV2.String() commp, err := commcidv2.CommPFromPCidV2(pieceCidV2) @@ -368,7 +375,7 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid r.reader = reader r.err = nil r.cancel = readerCtxCancel - r.pieceSize = size + r.rawSize = size } else { r = rr.(*cachedSectionReader) r.refs++ @@ -393,7 +400,7 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid return nil, 0, r.err } - rs := io.NewSectionReader(r.reader, 0, int64(r.pieceSize)) + rs := io.NewSectionReader(r.reader, 0, int64(r.rawSize)) return struct { io.Closer @@ -405,5 +412,5 @@ func (cpr *CachedPieceReader) GetSharedPieceReader(ctx context.Context, pieceCid Reader: rs, Seeker: rs, ReaderAt: r.reader, - }, r.pieceSize, nil + }, r.rawSize, nil } diff --git a/market/indexstore/cql/0002_piece_index.cql b/market/indexstore/cql/0002_piece_index.cql index b8bc6868b..91f2ef96d 100644 --- a/market/indexstore/cql/0002_piece_index.cql +++ b/market/indexstore/cql/0002_piece_index.cql @@ -4,4 +4,11 @@ CREATE TABLE IF NOT EXISTS PieceToAggregatePiece ( UnpaddedOffset BIGINT, UnpaddedLength BIGINT, PRIMARY KEY (PieceCid, AggregatePieceCid) +); + +CREATE TABLE IF NOT EXISTS PDPCacheLayer ( + PieceCid BLOB PRIMARY KEY, + LayerIndex INT, + Leaf BLOB, + LeafIndex BIGINT ); \ No newline at end of file diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index aebf887d8..7d1425a3e 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -582,3 +582,126 @@ func (i *IndexStore) UpdatePieceCidV1ToV2(ctx context.Context, pieceCidV1 cid.Ci return nil } + +type NodeDigest struct { + Layer int // Layer index in the merkle Tree + Index int64 // logical index at that layer + Hash [32]byte // 32 bytes +} + +func (i *IndexStore) AddPDPLayer(ctx context.Context, pieceCidV2 cid.Cid, layer []NodeDigest) error { + qry := `INSERT INTO PDPCacheLayer (PieceCid, LayerIndex, Leaf, LeafIndex) VALUES (?, ?, ?, ?)` + pieceCidBytes := pieceCidV2.Bytes() + var batch *gocql.Batch + batchSize := i.settings.InsertBatchSize + + if len(layer) == 0 { + return xerrors.Errorf("no records to insert") + } + + for _, r := range layer { + if batch == nil { + batch = i.session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + } + + batch.Entries = append(batch.Entries, gocql.BatchEntry{ + Stmt: qry, + Args: []interface{}{pieceCidBytes, r.Layer, r.Hash, r.Index}, + Idempotent: true, + }) + + if len(batch.Entries) >= batchSize { + if err := i.session.ExecuteBatch(batch); err != nil { + return xerrors.Errorf("executing batch insert for PDP cache layer for piece %s: %w", pieceCidV2.String(), err) + } + batch = nil + } + } + + if batch != nil { + if len(batch.Entries) >= 0 { + if err := i.session.ExecuteBatch(batch); err != nil { + return xerrors.Errorf("executing batch insert for PDP cache layer for piece %s: %w", pieceCidV2.String(), err) + } + } + } + + return nil +} + +func (i *IndexStore) GetPDPLayer(ctx context.Context, pieceCidV2 cid.Cid) ([]NodeDigest, error) { + var layer []NodeDigest + qry := `SELECT LayerIndex, Leaf, LeafIndex FROM PDPCacheLayer WHERE PieceCid = ? ORDER BY LeafIndex ASC` + iter := i.session.Query(qry, pieceCidV2.Bytes()).WithContext(ctx).Iter() + r := make([]byte, 32) + var idx int64 + var layerIdx int + for iter.Scan(&layerIdx, &r, &idx) { + layer = append(layer, NodeDigest{ + Layer: layerIdx, + Index: idx, + Hash: [32]byte(r), + }) + r = make([]byte, 32) + } + if err := iter.Close(); err != nil { + return nil, xerrors.Errorf("iterating PDP cache layer (P:0x%02x): %w", pieceCidV2.Bytes(), err) + } + return layer, nil +} + +func (i *IndexStore) DeletePDPLayer(ctx context.Context, pieceCidV2 cid.Cid) error { + qry := `DELETE FROM PDPCacheLayer WHERE PieceCid = ?` + if err := i.session.Query(qry, pieceCidV2.Bytes()).WithContext(ctx).Exec(); err != nil { + return xerrors.Errorf("deleting PDP cache layer (P:0x%02x): %w", pieceCidV2.Bytes(), err) + } + return nil +} + +func (i *IndexStore) HasPDPLayer(ctx context.Context, pieceCidV2 cid.Cid) (bool, error) { + qry := `SELECT Leaf FROM PDPCacheLayer WHERE PieceCid = ? LIMIT 1` + iter := i.session.Query(qry, pieceCidV2.Bytes()).WithContext(ctx).Iter() + + var hashes [][]byte + var r []byte + for iter.Scan(&r) { + if r != nil { + hashes = append(hashes, r) + r = make([]byte, 32) + } + } + if err := iter.Close(); err != nil { + return false, xerrors.Errorf("iterating PDP cache layer (P:0x%02x): %w", pieceCidV2.Bytes(), err) + } + + return len(hashes) > 0, nil + +} + +func (i *IndexStore) GetPDPNode(ctx context.Context, pieceCidV2 cid.Cid, index int64) (bool, *NodeDigest, error) { + qry := `SELECT IndexLayer, Leaf, LeafIndex FROM PDPCacheLayer WHERE PieceCid = ? AND LeafIndex = ? LIMIT 1` + iter := i.session.Query(qry, pieceCidV2.Bytes(), index).WithContext(ctx).Iter() + + var node *NodeDigest + + var r []byte + var idx int + var lidx int64 + for iter.Scan(&r, &idx, &lidx) { + if r != nil { + node = &NodeDigest{ + Layer: idx, + Index: lidx, + Hash: [32]byte(r), + } + r = make([]byte, 32) + } + } + if err := iter.Close(); err != nil { + return false, nil, xerrors.Errorf("iterating PDP cache layer (P:0x%02x): %w", pieceCidV2.Bytes(), err) + } + if node != nil { + return true, node, nil + } + return false, nil, nil +} diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 014c6b47c..8036e47d8 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -75,6 +75,7 @@ func Router(mdh *MK20DealHandler) http.Handler { mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "request timeout")) mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) + mux.Method("GET", "/update", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UpdateDeal), requestTimeout, "request timeout")) return mux } @@ -453,5 +454,79 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re return } - mdh.dm.MK20Handler.HandleUploadFinalize(id, w) + ct := r.Header.Get("Content-Type") + // If Content-Type is not set this is does not require updating the deal + if len(ct) == 0 { + log.Infow("received finalize upload proposal without content type", "id", id) + mdh.dm.MK20Handler.HandleUploadFinalize(id, nil, w) + return + } + + var deal mk20.Deal + if ct != "application/json" { + log.Errorf("invalid content type: %s", ct) + http.Error(w, "invalid content type", http.StatusBadRequest) + return + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("error reading request body: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + log.Infow("received upload finalize proposal", "body", string(body)) + + err = json.Unmarshal(body, &deal) + if err != nil { + log.Errorf("error unmarshaling json: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + mdh.dm.MK20Handler.HandleUploadFinalize(id, &deal, w) +} + +func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Request) { + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + ct := r.Header.Get("Content-Type") + var deal mk20.Deal + if ct != "application/json" { + log.Errorf("invalid content type: %s", ct) + http.Error(w, "invalid content type", http.StatusBadRequest) + return + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("error reading request body: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err = json.Unmarshal(body, &deal) + if err != nil { + log.Errorf("error unmarshaling json: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + log.Infow("received deal update proposal", "body", string(body)) + + mdh.dm.MK20Handler.UpdateDeal(id, &deal, w) } diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md index 56fb9e2fe..01b13b295 100644 --- a/market/mk20/http/info.md +++ b/market/mk20/http/info.md @@ -248,8 +248,12 @@ PDPV1 represents configuration for product-specific PDP version 1 deals. | Field | Type | Tag | Description | |-------|------|-----|-------------| -| ProofSetID | [uint64](https://pkg.go.dev/builtin#uint64) | json:"proof_set_id" | | -| DeleteRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_root" | DeleteRoot indicates whether the root of the data should be deleted. This basically means end of deal lifetime. | +| CreateProofSet | [bool](https://pkg.go.dev/builtin#bool) | json:"create_proof_set" | CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider. | +| DeleteProofSet | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_proof_set" | DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. ProofSetID must be defined. | +| AddRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"add_root" | AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined. | +| DeleteRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_root" | DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined. | +| ProofSetID | [*uint64](https://pkg.go.dev/builtin#uint64) | json:"proof_set_id" | ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. | +| ExtraData | [[]byte](https://pkg.go.dev/builtin#byte) | json:"extra_data" | ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. | ### PieceDataFormat diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index d84a7b120..66591a9da 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -2,15 +2,21 @@ package mk20 import ( "context" + "encoding/json" + "errors" "fmt" "net/http" "runtime" "runtime/debug" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/oklog/ulid" "github.com/samber/lo" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -74,6 +80,8 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } } + go markDownloaded(ctx, db) + return &MK20{ miners: miners, db: db, @@ -115,8 +123,11 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti log.Debugw("deal validated", "deal", deal.Identifier.String()) - return m.processDDODeal(ctx, deal) + if deal.Products.DDOV1 != nil { + return m.processDDODeal(ctx, deal) + } + return m.processPDPDeal(ctx, deal) } func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { @@ -309,6 +320,375 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe return nil, nil } +func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { + rejection, err := m.sanitizePDPDeal(ctx, deal) + if err != nil { + log.Errorw("PDP deal rejected", "deal", deal, "error", err) + return rejection + } + + log.Debugw("PDP deal sanitized", "deal", deal.Identifier.String()) + + if rejection != nil { + return rejection + } + + // Save deal to DB and start pipeline if required + comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // Save deal + err = deal.SaveToDB(tx) + if err != nil { + return false, xerrors.Errorf("saving deal to DB: %w", err) + } + + // If we have data source other that PUT then start the pipeline + if deal.Data != nil { + if deal.Data.SourceHttpPut != nil || deal.Data.SourceAggregate != nil { + err = insertPDPPipeline(ctx, tx, deal) + if err != nil { + return false, xerrors.Errorf("inserting pipeline: %w", err) + } + } + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorw("error inserting PDP deal into DB", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + if !comm { + log.Errorw("error committing PDP deal into DB", "deal", deal) + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusInternalServerError, + } + } + log.Debugw("PDP deal inserted in DB", "deal", deal.Identifier.String()) + return nil +} + +func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { + if deal.Data != nil { + if deal.Data.SourceOffline != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: http.StatusBadRequest, + Reason: "Offline data source is not supported for pdp_v1", + }, nil + } + } + return nil, nil +} + +func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error { + pdp := deal.Products.PDPV1 + data := deal.Data + dealID := deal.Identifier.String() + pi, err := deal.PieceInfo() + if err != nil { + return fmt.Errorf("getting piece info: %w", err) + } + + aggregation := 0 + if data.Format.Aggregate != nil { + aggregation = int(data.Format.Aggregate.Type) + } + + // Insert pipeline when Data source is HTTP + if data.SourceHTTP != nil { + var pieceID int64 + // Attempt to select the piece ID first + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, pi.PieceCIDV1.String(), pi.Size).Scan(&pieceID) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // Piece does not exist, attempt to insert + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, TRUE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id`, pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&pieceID) + if err != nil { + return xerrors.Errorf("inserting new parked piece and getting id: %w", err) + } + } else { + // Some other error occurred during select + return xerrors.Errorf("checking existing parked piece: %w", err) + } + } + + var refIds []int64 + + // Add parked_piece_refs + for _, src := range data.SourceHTTP.URLs { + var refID int64 + + headers, err := json.Marshal(src.Headers) + if err != nil { + return xerrors.Errorf("marshaling headers: %w", err) + } + + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + VALUES ($1, $2, $3, TRUE) RETURNING ref_id`, pieceID, src.URL, headers).Scan(&refID) + if err != nil { + return xerrors.Errorf("inserting parked piece ref: %w", err) + } + refIds = append(refIds, refID) + } + + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) VALUES ($1, $2, $3, $4, $5)`, + dealID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1, refIds) + if err != nil { + return xerrors.Errorf("inserting PDP download pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting PDP download pipeline: %d rows affected", n) + } + + n, err = tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, + extra_data, deal_aggregation) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + dealID, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, aggregation) + if err != nil { + return xerrors.Errorf("inserting PDP pipeline: %w", err) + } + if n != 1 { + return xerrors.Errorf("inserting PDP pipeline: %d rows affected", n) + } + return nil + } + + // Insert pipeline when data source is aggregate + if deal.Data.SourceAggregate != nil { + + // Find all unique pieces where data source is HTTP + type downloadkey struct { + ID string + PieceCID cid.Cid + Size abi.PaddedPieceSize + RawSize uint64 + } + toDownload := make(map[downloadkey][]HttpUrl) + + for _, piece := range deal.Data.SourceAggregate.Pieces { + spi, err := GetPieceInfo(piece.PieceCID) + if err != nil { + return xerrors.Errorf("getting piece info: %w", err) + } + if piece.SourceHTTP != nil { + urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] + if ok { + toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) + } else { + toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs + } + } + } + + batch := &pgx.Batch{} + batchSize := 5000 + + for k, v := range toDownload { + for _, src := range v { + headers, err := json.Marshal(src.Headers) + if err != nil { + return xerrors.Errorf("marshal headers: %w", err) + } + batch.Queue(`WITH inserted_piece AS ( + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) + VALUES ($1, $2, $3, FALSE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id + ), + selected_piece AS ( + SELECT COALESCE( + (SELECT id FROM inserted_piece), + (SELECT id FROM parked_pieces + WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = FALSE AND cleanup_task_id IS NULL) + ) AS id + ), + inserted_ref AS ( + INSERT INTO parked_piece_refs (piece_id, data_url, data_headers, long_term) + SELECT id, $4, $5, FALSE FROM selected_piece + RETURNING ref_id + ) + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) + VALUES ($6, $1, $2, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) + ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE + SET ref_ids = array_append( + market_mk20_download_pipeline.ref_ids, + (SELECT ref_id FROM inserted_ref) + ) + WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, + k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, ProductNamePDPV1) + } + + if batch.Len() > batchSize { + res := tx.SendBatch(ctx, batch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece query batch: %w", err) + } + batch = &pgx.Batch{} + } + } + + if batch.Len() > 0 { + res := tx.SendBatch(ctx, batch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing parked piece query batch: %w", err) + } + } + + pBatch := &pgx.Batch{} + pBatchSize := 4000 + for i, piece := range deal.Data.SourceAggregate.Pieces { + spi, err := GetPieceInfo(piece.PieceCID) + if err != nil { + return xerrors.Errorf("getting piece info: %w", err) + } + pBatch.Queue(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, + proof_set_id, extra_data, piece_ref, deal_aggregation, aggr_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + dealID, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), spi.Size, spi.RawSize, + pdp.ExtraData, *pdp.ProofSetID, aggregation, i) + if pBatch.Len() > pBatchSize { + res := tx.SendBatch(ctx, pBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 pipeline insert batch: %w", err) + } + pBatch = &pgx.Batch{} + } + } + if pBatch.Len() > 0 { + res := tx.SendBatch(ctx, pBatch) + if err := res.Close(); err != nil { + return xerrors.Errorf("closing mk20 pipeline insert batch: %w", err) + } + } + return nil + } + + return xerrors.Errorf("unknown data source type") +} + +func markDownloaded(ctx context.Context, db *harmonydb.DB) { + md := func(ctx context.Context, db *harmonydb.DB) { + var deals []struct { + ID string `db:"id"` + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + } + + err := db.Select(ctx, &deals, `SELECT id, piece_cid, piece_size FROM pdp_pipeline WHERE piece_ref IS NULL`) + if err != nil { + log.Errorw("error getting PDP deals", "error", err) + } + + for _, deal := range deals { + _, err = db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var refid int64 + err = tx.QueryRow(`SELECT u.ref_id FROM ( + SELECT unnest(dp.ref_ids) AS ref_id + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + ) u + JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.complete = TRUE + LIMIT 1;`, deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1).Scan(&refid) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) + } + + // Remove other ref_ids from piece_park_refs + _, err = tx.Exec(`DELETE FROM parked_piece_refs + WHERE ref_id IN ( + SELECT unnest(dp.ref_ids) + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + ) + AND ref_id != $5;`, deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1, refid) + if err != nil { + return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, + deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1) + if err != nil { + return false, xerrors.Errorf("failed to delete piece from download table: %w", err) + } + + _, err = tx.Exec(`UPDATE pdp_pipeline SET downloaded = TRUE, piece_ref = $1 + WHERE id = $2 + AND piece_cid = $3 + AND piece_size = $4`, + refid, deal.ID, deal.PieceCID, deal.PieceSize) + if err != nil { + return false, xerrors.Errorf("failed to update download statos for PDP pipeline: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorw("error updating PDP deal", "deal", deal, "error", err) + } + } + } + + ticker := time.NewTicker(time.Second * 2) + defer ticker.Stop() + for { + select { + case <-ticker.C: + md(ctx, db) + case <-ctx.Done(): + return + } + } +} + +func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, w http.ResponseWriter) { + ctx := context.Background() + var exists bool + err := m.db.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_deal + WHERE id = $1)`, id.String()).Scan(&exists) + if err != nil { + log.Errorw("failed to check if deal exists", "deal", id, "error", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + + if !exists { + http.Error(w, "", http.StatusNotFound) + return + } + + if deal == nil { + http.Error(w, "deal not defined", int(ErrBadProposal)) + } + + code, err := m.updateDealDetails(id, deal) + if err != nil { + log.Errorw("failed to update deal details", "deal", id, "error", err) + if code == http.StatusInternalServerError { + http.Error(w, "", http.StatusInternalServerError) + } else { + http.Error(w, err.Error(), int(code)) + } + return + } + + w.WriteHeader(http.StatusOK) +} + // To be used later for when data source is minerID //func validateMinerAddresses(madrs []abi.Multiaddrs, pcid cid.Cid, psize abi.PaddedPieceSize, rawSize int64) bool { // var surls []*url.URL diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index dad677151..b238e7091 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -386,7 +386,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } -func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { +func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { ctx := context.Background() var exists bool err := m.db.QueryRow(ctx, `SELECT EXISTS ( @@ -405,6 +405,21 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { return } + if deal != nil { + // This is a deal where DataSource was not set - we should update the deal + code, err := m.updateDealDetails(id, deal) + if err != nil { + log.Errorw("failed to update deal details", "deal", id, "error", err) + if code == http.StatusInternalServerError { + http.Error(w, "", http.StatusInternalServerError) + } else { + http.Error(w, err.Error(), int(code)) + } + return + } + } + + // Now update the upload status to trigger the correct pipeline n, err := m.db.Exec(ctx, `UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) if err != nil { log.Errorw("failed to finalize deal upload", "deal", id, "error", err) @@ -421,63 +436,42 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, w http.ResponseWriter) { w.WriteHeader(http.StatusOK) } -//func (m *MK20) MarkChunkComplete(ctx context.Context) { -// ticker := time.NewTicker(time.Second * 3) -// defer ticker.Stop() -// for { -// select { -// case <-ticker.C: -// markChunksComplete(ctx, m.db) -// case <-ctx.Done(): -// return -// } -// } -//} -// -//func markChunksComplete(ctx context.Context, db *harmonydb.DB) { -// var chunks []struct { -// ID string `db:"id"` -// Chunk int `db:"chunk"` -// ChunkSize int64 `db:"chunk_size"` -// Complete bool `db:"complete"` -// RefId int64 `db:"ref_id"` -// } -// -// err := db.Select(ctx, &chunks, `SELECT id, -// chunk, -// chunk_size, -// ref_id, -// complete -// FROM market_mk20_deal_chunk -// WHERE finalize = FALSE -// AND complete = FALSE -// AND ref_id IS NOT NULL`) -// if err != nil { -// log.Errorw("failed to get chunks to mark complete", "error", err) -// return -// } -// for _, chunk := range chunks { -// var complete bool -// err := db.QueryRow(ctx, `SELECT p.complete -// FROM parked_pieces AS p -// JOIN parked_piece_refs AS r -// ON r.piece_id = p.id -// WHERE r.ref_id = $1`, chunk.RefId).Scan(&complete) -// if err != nil { -// log.Errorw("failed to get piece complete status", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) -// continue -// } -// if complete { -// _, err := db.Exec(ctx, `UPDATE market_mk20_deal_chunk -// SET complete = TRUE -// WHERE id = $1 -// AND chunk = $2 -// AND ref_id = $3 -// AND finalize = FALSE`, chunk.ID, chunk.Chunk, chunk.RefId) -// if err != nil { -// log.Errorw("failed to mark chunk complete", "id", chunk.ID, "chunk", chunk.Chunk, "error", err) -// continue -// } -// } -// } -//} +func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (ErrorCode, error) { + ctx := context.Background() // Let's not use request context to avoid DB inconsistencies + + if deal.Identifier.Compare(id) != 0 { + return ErrBadProposal, xerrors.Errorf("deal ID and proposal ID do not match") + } + + // Validate the deal + code, err := deal.Validate(m.db, &m.cfg.Market.StorageMarketConfig.MK20) + if err != nil { + return code, err + } + + log.Debugw("deal validated", "deal", deal.Identifier.String()) + + // Verify we have a deal is DB + var exists bool + err = m.db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_deal WHERE id = $1)`, id.String()).Scan(&exists) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("failed to check if deal exists: %w", err) + } + + if !exists { + return http.StatusNotFound, xerrors.Errorf("deal not found") + } + + // Get updated deal + ndeal, code, err := UpdateDealDetails(ctx, m.db, id, deal, &m.cfg.Market.StorageMarketConfig.MK20) + if err != nil { + return code, err + } + + // Save the updated deal to DB + err = ndeal.UpdateDeal(ctx, m.db) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("failed to update deal: %w", err) + } + return http.StatusOK, nil +} diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index f0d0483e6..8f3cf06e4 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -1,16 +1,35 @@ package mk20 import ( + "context" + "net/http" + + "golang.org/x/xerrors" + "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" ) // PDPV1 represents configuration for product-specific PDP version 1 deals. type PDPV1 struct { - ProofSetID uint64 `json:"proof_set_id"` + // CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider. + CreateProofSet bool `json:"create_proof_set"` + + // DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. + // ProofSetID must be defined. + DeleteProofSet bool `json:"delete_proof_set"` + + // AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined. + AddRoot bool `json:"add_root"` - // DeleteRoot indicates whether the root of the data should be deleted. This basically means end of deal lifetime. + // DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined. DeleteRoot bool `json:"delete_root"` + + // ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. + ProofSetID *uint64 `json:"proof_set_id"` + + // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. + ExtraData []byte `json:"extra_data"` } func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { @@ -18,9 +37,92 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e if err != nil { return code, err } + + if ok := p.CreateProofSet || p.DeleteProofSet || p.AddRoot || p.DeleteRoot; !ok { + return ErrBadProposal, xerrors.Errorf("deal must have one of the following flags set: create_proof_set, delete_proof_set, add_root, delete_root") + } + + if p.CreateProofSet && p.ProofSetID != nil { + return ErrBadProposal, xerrors.Errorf("create_proof_set cannot be set with proof_set_id") + } + + if p.DeleteProofSet && p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") + } + + if p.AddRoot && p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") + } + + if p.DeleteRoot && p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") + } + + // Only 1 action is allowed per deal + if btoi(p.CreateProofSet)+btoi(p.DeleteProofSet)+btoi(p.AddRoot)+btoi(p.DeleteRoot) > 1 { + return ErrBadProposal, xerrors.Errorf("only one action is allowed per deal") + } + + if p.CreateProofSet { + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for create_proof_set") + } + } + + ctx := context.Background() + + if p.DeleteProofSet { + pid := *p.ProofSetID + var exists bool + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + } + if !exists { + return ErrBadProposal, xerrors.Errorf("proofset does not exist") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_proof_set") + } + } + + if p.AddRoot { + pid := *p.ProofSetID + var exists bool + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + } + if !exists { + return ErrBadProposal, xerrors.Errorf("proofset does not exist") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for add_root") + } + } + + if p.DeleteRoot { + pid := *p.ProofSetID + var exists bool + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + if err != nil { + return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") + } + } + return Ok, nil } +func btoi(b bool) int { + if b { + return 1 + } + return 0 +} + func (p *PDPV1) ProductName() ProductName { return ProductNamePDPV1 } diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 8c9ee7316..325955fb5 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -43,7 +43,13 @@ func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, er return code, xerrors.Errorf("products validation failed: %w", err) } - return d.Data.Validate(db) + // Validate data if present + if d.Data != nil { + return d.Data.Validate(db) + } + + // Return without validating data for initial phase of /Put deals or PDP Delete deals + return Ok, nil } func (d *Deal) ValidateSignature() (ErrorCode, error) { @@ -291,11 +297,19 @@ func GetPieceInfo(c cid.Cid) (*PieceInfo, error) { } func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { + var nproducts int if d.DDOV1 != nil { + nproducts++ code, err := d.DDOV1.Validate(db, cfg) if err != nil { return code, err } + if d.RetrievalV1 == nil { + return ErrProductValidationFailed, xerrors.Errorf("retrieval v1 is required for ddo v1") + } + if d.RetrievalV1.AnnouncePiece { + return ErrProductValidationFailed, xerrors.Errorf("announce piece is not supported for ddo v1") + } } if d.RetrievalV1 != nil { code, err := d.RetrievalV1.Validate(db, cfg) @@ -304,11 +318,27 @@ func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, } } if d.PDPV1 != nil { + nproducts++ code, err := d.PDPV1.Validate(db, cfg) if err != nil { return code, err } + if d.RetrievalV1 == nil { + return ErrProductValidationFailed, xerrors.Errorf("retrieval v1 is required for pdp v1") + } + if d.RetrievalV1.Indexing || d.RetrievalV1.AnnouncePayload { + return ErrProductValidationFailed, xerrors.Errorf("payload indexing and announcement is not supported for pdp v1") + } + } + + if nproducts == 0 { + return ErrProductValidationFailed, xerrors.Errorf("no products defined") } + + if d.DDOV1 != nil && d.PDPV1 != nil { + return ErrProductValidationFailed, xerrors.Errorf("ddo_v1 and pdp_v1 are mutually exclusive") + } + return Ok, nil } @@ -432,7 +462,7 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { return nil } -func (d *Deal) UpdateDeal(tx *harmonydb.Tx) error { +func (d *Deal) UpdateDealWithTx(tx *harmonydb.Tx) error { dbDeal, err := d.ToDBDeal() if err != nil { return xerrors.Errorf("to db deal: %w", err) @@ -449,10 +479,35 @@ func (d *Deal) UpdateDeal(tx *harmonydb.Tx) error { pdp_v1 = $8`, dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, dbDeal.RawSize, dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) if err != nil { - return xerrors.Errorf("insert deal: %w", err) + return xerrors.Errorf("update deal: %w", err) } if n != 1 { - return xerrors.Errorf("insert deal: expected 1 row affected, got %d", n) + return xerrors.Errorf("update deal: expected 1 row affected, got %d", n) + } + return nil +} + +func (d *Deal) UpdateDeal(ctx context.Context, db *harmonydb.DB) error { + dbDeal, err := d.ToDBDeal() + if err != nil { + return xerrors.Errorf("to db deal: %w", err) + } + + n, err := db.Exec(ctx, `UPDATE market_mk20_deal SET + piece_cid_v2 = $1, + piece_cid = $2, + piece_size = $3, + raw_size = $4, + data = $5, + ddo_v1 = $6, + retrieval_v1 = $7, + pdp_v1 = $8`, dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, dbDeal.RawSize, + dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) + if err != nil { + return xerrors.Errorf("update deal: %w", err) + } + if n != 1 { + return xerrors.Errorf("update deal: expected 1 row affected, got %d", n) } return nil } @@ -696,3 +751,35 @@ type UploadStatus struct { //MissingChunks is a slice containing the indices of missing chunks. MissingChunks []int `json:"missing_chunks"` } + +func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config) (*Deal, ErrorCode, error) { + ddeal, err := DealFromDB(ctx, db, id) + if err != nil { + return nil, http.StatusInternalServerError, xerrors.Errorf("getting deal from DB: %w", err) + } + + // Run the following checks + // If Data details exist, do not update them + // If DDOV1 is defined then no update to it + // If PDPV1 is defined then no update to it + // If PDPv1 is defined by DDOV1 is not, then allow updating it + // If DDOV1 is defined then don't allow PDPv1 yet + + if ddeal.Data == nil { + ddeal.Data = deal.Data + } + + if ddeal.Products.DDOV1 == nil || deal.Products.DDOV1 != nil { + return nil, ErrBadProposal, xerrors.Errorf("ddov1 update is not yet supported") + } + + if ddeal.Products.RetrievalV1 == nil || deal.Products.RetrievalV1 != nil { + ddeal.Products.RetrievalV1 = deal.Products.RetrievalV1 + } + + code, err := ddeal.Validate(db, cfg) + if err != nil { + return nil, code, xerrors.Errorf("validate deal: %w", err) + } + return ddeal, Ok, nil +} diff --git a/pdp/contract/types.go b/pdp/contract/types.go new file mode 100644 index 000000000..613da7996 --- /dev/null +++ b/pdp/contract/types.go @@ -0,0 +1,9 @@ +package contract + +import "math/big" + +// RootData matches the Solidity RootData struct +type RootData struct { + Root struct{ Data []byte } + RawSize *big.Int +} diff --git a/pdp/handlers.go b/pdp/handlers.go index 75d06b250..79c73664d 100644 --- a/pdp/handlers.go +++ b/pdp/handlers.go @@ -65,7 +65,7 @@ func NewPDPService(db *harmonydb.DB, stor paths.StashStore, ec *ethclient.Client func Routes(p *PDPService) http.Handler { r := chi.NewRouter() - + // Routes for proof sets r.Route("/proof-sets", func(r chi.Router) { // POST /pdp/proof-sets - Create a new proof set diff --git a/tasks/pdp/proofset_addroot_watch.go b/tasks/pdp/proofset_addroot_watch.go index db042dd5f..fcb32428d 100644 --- a/tasks/pdp/proofset_addroot_watch.go +++ b/tasks/pdp/proofset_addroot_watch.go @@ -8,10 +8,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/chainsched" + "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/pdp/contract" chainTypes "github.com/filecoin-project/lotus/chain/types" @@ -19,8 +21,13 @@ import ( // Structures to represent database records type ProofSetRootAdd struct { - ProofSet uint64 `db:"proofset"` - AddMessageHash string `db:"add_message_hash"` + ID string `db:"id"` + Client string `db:"client"` + PieceCID string `db:"piece_cid"` // pieceCIDV2 + ProofSet uint64 `db:"proofset"` + PieceRef int64 `db:"piece_ref"` + AddMessageHash string `db:"add_message_hash"` + AddMessageIndex int64 `db:"add_message_index"` } // RootAddEntry represents entries from pdp_proofset_root_adds @@ -57,9 +64,9 @@ func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB, ethCl var rootAdds []ProofSetRootAdd err := db.Select(ctx, &rootAdds, ` - SELECT DISTINCT proofset, add_message_hash - FROM pdp_proofset_root_adds - WHERE add_message_ok = TRUE AND roots_added = FALSE + SELECT id, client, piece_cid, proofset, piece_ref, add_message_hash, add_message_index + FROM pdp_pipeline + WHERE after_add_root = TRUE AND after_add_root_msg = FALSE `) if err != nil { return xerrors.Errorf("failed to select proof set root adds: %w", err) @@ -85,11 +92,12 @@ func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB, ethCl func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client, rootAdd ProofSetRootAdd) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte + var txSuccess bool err := db.QueryRow(ctx, ` - SELECT tx_receipt + SELECT tx_success, tx_receipt FROM message_waits_eth WHERE signed_tx_hash = $1 - `, rootAdd.AddMessageHash).Scan(&txReceiptJSON) + `, rootAdd.AddMessageHash).Scan(&txSuccess, &txReceiptJSON) if err != nil { return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", rootAdd.AddMessageHash, err) } @@ -101,16 +109,46 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *et return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", rootAdd.AddMessageHash, err) } - // Parse the logs to extract root IDs and other data - err = extractAndInsertRootsFromReceipt(ctx, db, &txReceipt, rootAdd) - if err != nil { - return xerrors.Errorf("failed to extract roots from receipt for tx %s: %w", rootAdd.AddMessageHash, err) + if !txSuccess { + // This means msg failed, we should let the user know + // TODO: Review if error would be in receipt + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", rootAdd.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, rootAdd.ID) + if err != nil { + return false, xerrors.Errorf("failed to clean up pdp pipeline: %w", err) + } + return true, nil + }) + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + return nil } - return nil -} + pcid, err := cid.Parse(rootAdd.PieceCID) + if err != nil { + return xerrors.Errorf("failed to parse piece CID: %w", err) + } + pi, err := mk20.GetPieceInfo(pcid) + if err != nil { + return xerrors.Errorf("failed to get piece info: %w", err) + } -func extractAndInsertRootsFromReceipt(ctx context.Context, db *harmonydb.DB, receipt *types.Receipt, rootAdd ProofSetRootAdd) error { // Get the ABI from the contract metadata pdpABI, err := contract.PDPVerifierMetaData.GetAbi() if err != nil { @@ -127,7 +165,7 @@ func extractAndInsertRootsFromReceipt(ctx context.Context, db *harmonydb.DB, rec eventFound := false // Iterate over the logs in the receipt - for _, vLog := range receipt.Logs { + for _, vLog := range txReceipt.Logs { // Check if the log corresponds to the RootsAdded event if len(vLog.Topics) > 0 && vLog.Topics[0] == event.ID { // The setId is an indexed parameter in Topics[1], but we don't need it here @@ -165,69 +203,72 @@ func extractAndInsertRootsFromReceipt(ctx context.Context, db *harmonydb.DB, rec return fmt.Errorf("RootsAdded event not found in receipt") } - // Now we have the firstAdded rootId, proceed with database operations - - // Begin a database transaction - _, err = db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - // Fetch the entries from pdp_proofset_root_adds - var rootAddEntries []RootAddEntry - err := tx.Select(&rootAddEntries, ` - SELECT proofset, root, add_message_hash, add_message_index, subroot, subroot_offset, subroot_size, pdp_pieceref - FROM pdp_proofset_root_adds - WHERE proofset = $1 AND add_message_hash = $2 - ORDER BY add_message_index ASC, subroot_offset ASC - `, rootAdd.ProofSet, rootAdd.AddMessageHash) + rootId := rootIds[rootAdd.AddMessageIndex] + + // Insert into message_waits_eth and pdp_proofset_roots + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + // Update proof set for initialization upon first add + _, err = tx.Exec(` + UPDATE pdp_proof_sets SET init_ready = true + WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL + `, rootAdd.ProofSet) if err != nil { - return false, fmt.Errorf("failed to select from pdp_proofset_root_adds: %w", err) + return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) } - // For each entry, use the corresponding rootId from the event - for _, entry := range rootAddEntries { - if entry.AddMessageIndex >= uint64(len(rootIds)) { - return false, fmt.Errorf("index out of bounds: entry index %d exceeds rootIds length %d", - entry.AddMessageIndex, len(rootIds)) - } - - rootId := rootIds[entry.AddMessageIndex] - // Insert into pdp_proofset_roots - _, err := tx.Exec(` - INSERT INTO pdp_proofset_roots ( - proofset, - root, - root_id, - subroot, - subroot_offset, - subroot_size, - pdp_pieceref, - add_message_hash, - add_message_index - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9 - ) - `, entry.ProofSet, entry.Root, rootId, entry.Subroot, entry.SubrootOffset, entry.SubrootSize, entry.PDPPieceRefID, entry.AddMessageHash, entry.AddMessageIndex) - if err != nil { - return false, fmt.Errorf("failed to insert into pdp_proofset_roots: %w", err) - } + // Insert into pdp_proofset_roots + n, err := tx.Exec(` + INSERT INTO pdp_proofset_root ( + proofset, + client, + piece_cid_v2, + piece_cid, + piece_size, + raw_size, + root, + piece_ref, + add_deal_id, + add_message_hash, + add_message_index + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + `, + rootAdd.ProofSet, + rootAdd.Client, + pcid.String(), + pi.PieceCIDV1.String(), + pi.Size, + pi.RawSize, + rootId, + rootAdd.PieceRef, + rootAdd.ID, + rootAdd.AddMessageHash, + rootAdd.AddMessageIndex, + ) + if err != nil { + return false, xerrors.Errorf("failed to insert into pdp_proofset_root: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows inserted for pdp_proofset_root: %d", n) } - // Mark as processed in pdp_proofset_root_adds (don't delete, for transaction tracking) - rowsAffected, err := tx.Exec(` - UPDATE pdp_proofset_root_adds - SET roots_added = TRUE - WHERE proofset = $1 AND add_message_hash = $2 AND roots_added = FALSE - `, rootAdd.ProofSet, rootAdd.AddMessageHash) + n, err = tx.Exec(`UPDATE pdp_pipeline SET after_add_root_msg = TRUE WHERE id = $1`, rootAdd.ID) if err != nil { - return false, fmt.Errorf("failed to update pdp_proofset_root_adds: %w", err) + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } - - if int(rowsAffected) != len(rootAddEntries) { - return false, fmt.Errorf("expected to update %d rows in pdp_proofset_root_adds but updated %d", len(rootAddEntries), rowsAffected) + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_pipeline: %d", n) } + // Return true to commit the transaction return true, nil - }) + }, harmonydb.OptionRetry()) if err != nil { - return fmt.Errorf("failed to process root additions in DB: %w", err) + return xerrors.Errorf("failed to save details to DB: %w", err) + } + + if !comm { + return xerrors.Errorf("failed to commit transaction") } return nil diff --git a/tasks/pdp/proofset_create_watch.go b/tasks/pdp/proofset_create_watch.go index 8f54737ad..525201dda 100644 --- a/tasks/pdp/proofset_create_watch.go +++ b/tasks/pdp/proofset_create_watch.go @@ -19,8 +19,9 @@ import ( ) type ProofSetCreate struct { - CreateMessageHash string `db:"create_message_hash"` - Service string `db:"service"` + CreateMessageHash string `db:"tx_hash"` + ID string `db:"id"` + Client string `db:"client"` } func NewWatcherCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { @@ -40,10 +41,9 @@ func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethCli var proofSetCreates []ProofSetCreate err := db.Select(ctx, &proofSetCreates, ` - SELECT create_message_hash, service - FROM pdp_proofset_creates - WHERE ok = TRUE AND proofset_created = FALSE - `) + SELECT id, client, tx_hash, + FROM pdp_proof_set_create + WHERE tx_hash IS NOT NULL`) if err != nil { return xerrors.Errorf("failed to select proof set creates: %w", err) } @@ -68,11 +68,12 @@ func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethCli func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCreate, ethClient *ethclient.Client) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte + var txSuccess bool err := db.QueryRow(ctx, ` - SELECT tx_receipt + SELECT tx_success, tx_receipt FROM message_waits_eth WHERE signed_tx_hash = $1 - `, psc.CreateMessageHash).Scan(&txReceiptJSON) + `, psc.CreateMessageHash).Scan(&txReceiptJSON, &txSuccess) if err != nil { return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psc.CreateMessageHash, err) } @@ -84,6 +85,38 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", psc.CreateMessageHash, err) } + // Exit early if transaction executed with failure + if !txSuccess { + // This means msg failed, we should let the user know + // TODO: Review if error would be in receipt + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", psc.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`DELETE FROM pdp_proof_set_create WHERE id = $1`, psc.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete pdp_proof_set_create: %w", err) + } + return true, nil + }) + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + return nil + } + // Parse the logs to extract the proofSetId proofSetId, err := extractProofSetIdFromReceipt(&txReceipt) if err != nil { @@ -108,20 +141,37 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr return xerrors.Errorf("failed to get max proving period: %w", err) } - // Insert a new entry into pdp_proof_sets - err = insertProofSet(ctx, db, psc.CreateMessageHash, proofSetId, psc.Service, provingPeriod, challengeWindow) - if err != nil { - return xerrors.Errorf("failed to insert proof set %d for tx %+v: %w", proofSetId, psc, err) - } + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`INSERT INTO pdp_proof_set (id, client, proving_period, challenge_window, create_deal_id, create_message_hash) + VALUES ($1, $2, $3, $4, $5, $6)`, proofSetId, psc.Client, provingPeriod, challengeWindow, psc.ID, psc.CreateMessageHash) + if err != nil { + return false, xerrors.Errorf("failed to insert pdp_proof_set_create: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be inserted, got %d", n) + } + + n, err = tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, "Transaction failed", psc.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } - // Update pdp_proofset_creates to set proofset_created = TRUE - _, err = db.Exec(ctx, ` - UPDATE pdp_proofset_creates - SET proofset_created = TRUE - WHERE create_message_hash = $1 - `, psc.CreateMessageHash) + _, err = tx.Exec(`DELETE FROM pdp_proof_set_create WHERE id = $1`, psc.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete pdp_proof_set_create: %w", err) + } + return true, nil + }) if err != nil { - return xerrors.Errorf("failed to update proofset_creates for tx %s: %w", psc.CreateMessageHash, err) + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") } return nil @@ -152,16 +202,6 @@ func extractProofSetIdFromReceipt(receipt *types.Receipt) (uint64, error) { return 0, xerrors.Errorf("ProofSetCreated event not found in receipt") } -func insertProofSet(ctx context.Context, db *harmonydb.DB, createMsg string, proofSetId uint64, service string, provingPeriod uint64, challengeWindow uint64) error { - // Implement the insertion into pdp_proof_sets table - // Adjust the SQL statement based on your table schema - _, err := db.Exec(ctx, ` - INSERT INTO pdp_proof_sets (id, create_message_hash, service, proving_period, challenge_window) - VALUES ($1, $2, $3, $4, $5) - `, proofSetId, createMsg, service, provingPeriod, challengeWindow) - return err -} - func getProvingPeriodChallengeWindow(ctx context.Context, ethClient *ethclient.Client, listenerAddr common.Address) (uint64, uint64, error) { // ProvingPeriod schedule, err := contract.NewIPDPProvingSchedule(listenerAddr, ethClient) diff --git a/tasks/pdp/task_add_proofset.go b/tasks/pdp/task_add_proofset.go new file mode 100644 index 000000000..ca0e9f3f5 --- /dev/null +++ b/tasks/pdp/task_add_proofset.go @@ -0,0 +1,183 @@ +package pdp + +import ( + "context" + "database/sql" + "errors" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/pdp/contract" + "github.com/filecoin-project/curio/tasks/message" +) + +type PDPTaskAddProofSet struct { + db *harmonydb.DB + sender *message.SenderETH + ethClient *ethclient.Client + filClient PDPServiceNodeApi +} + +func NewPDPTaskAddProofSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskAddProofSet { + return &PDPTaskAddProofSet{ + db: db, + sender: sender, + ethClient: ethClient, + filClient: filClient, + } +} + +func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + var pcreates []struct { + RecordKeeper string `db:"record_keeper"` + ExtraData []byte `db:"extra_data"` + } + + err = p.db.Select(ctx, &pcreates, `SELECT record_keeper, extra_data FROM pdp_proof_set_create WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to get task details from DB: %w", err) + } + + if len(pcreates) != 0 { + return false, xerrors.Errorf("incorrect rows for proofset create found for taskID %d", taskID) + } + + pcreate := pcreates[0] + + recordKeeperAddr := common.HexToAddress(pcreate.RecordKeeper) + if recordKeeperAddr == (common.Address{}) { + return false, xerrors.Errorf("invalid record keeper address: %s", pcreate.RecordKeeper) + } + + extraDataBytes := []byte{} + + if pcreate.ExtraData != nil { + extraDataBytes = pcreate.ExtraData + } + + // Get the sender address from 'eth_keys' table where role = 'pdp' limit 1 + fromAddress, err := p.getSenderAddress(ctx) + if err != nil { + return false, xerrors.Errorf("failed to get sender address: %w", err) + } + + // Manually create the transaction without requiring a Signer + // Obtain the ABI of the PDPVerifier contract + abiData, err := contract.PDPVerifierMetaData.GetAbi() + if err != nil { + return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) + } + + // Pack the method call data + data, err := abiData.Pack("createProofSet", recordKeeperAddr, extraDataBytes) + if err != nil { + return false, xerrors.Errorf("packing data: %w", err) + } + + // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) + tx := types.NewTransaction( + 0, + contract.ContractAddresses().PDPVerifier, + contract.SybilFee(), + 0, + nil, + data, + ) + + // Send the transaction using SenderETH + reason := "pdp-mkproofset" + txHash, err := p.sender.Send(ctx, fromAddress, tx, reason) + if err != nil { + return false, xerrors.Errorf("sending transaction: %w", err) + } + + // Insert into message_waits_eth and pdp_proofset_creates + txHashLower := strings.ToLower(txHash.Hex()) + n, err := p.db.Exec(ctx, `UPDATE pdp_proof_set_create SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_create: %d", n) + } + return true, nil +} + +func (p *PDPTaskAddProofSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + //TODO implement me + panic("implement me") +} + +func (p *PDPTaskAddProofSet) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "PDPAddProofSet", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(3*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return p.schedule(context.Background(), taskFunc) + }), + } +} + +func (p *PDPTaskAddProofSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_proof_set_create WHERE task_id IS NULL AND tx_hash IS NULL`).Scan(&id) + if err != nil { + return false, xerrors.Errorf("failed to query pdp_proof_set_create: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid id found for taskID") + } + + _, err = tx.Exec(`UPDATE pdp_proof_set_create SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +// getSenderAddress retrieves the sender address from the database where role = 'pdp' limit 1 +func (p *PDPTaskAddProofSet) getSenderAddress(ctx context.Context) (common.Address, error) { + // TODO: Update this function + var addressStr string + err := p.db.QueryRow(ctx, `SELECT address FROM eth_keys WHERE role = 'pdp' LIMIT 1`).Scan(&addressStr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return common.Address{}, errors.New("no sender address with role 'pdp' found") + } + return common.Address{}, err + } + address := common.HexToAddress(addressStr) + return address, nil +} + +func (p *PDPTaskAddProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &PDPTaskAddProofSet{} diff --git a/tasks/pdp/task_addroot.go b/tasks/pdp/task_addroot.go index c9b25a42a..bae4a2f54 100644 --- a/tasks/pdp/task_addroot.go +++ b/tasks/pdp/task_addroot.go @@ -1,190 +1,264 @@ package pdp -//import ( -// "context" -// "database/sql" -// "errors" -// "math/big" -// "net/http" -// "time" -// -// "github.com/ethereum/go-ethereum/common" -// "github.com/ethereum/go-ethereum/core/types" -// "github.com/ethereum/go-ethereum/ethclient" -// "github.com/filecoin-project/curio/harmony/harmonydb" -// "github.com/filecoin-project/curio/harmony/harmonytask" -// "github.com/filecoin-project/curio/harmony/resources" -// "github.com/filecoin-project/curio/harmony/taskhelp" -// "github.com/filecoin-project/curio/lib/passcall" -// "github.com/filecoin-project/curio/pdp/contract" -// "github.com/filecoin-project/curio/tasks/message" -// types2 "github.com/filecoin-project/lotus/chain/types" -// "golang.org/x/xerrors" -//) -// -//type PDPServiceNodeApi interface { -// ChainHead(ctx context.Context) (*types2.TipSet, error) -//} -// -//type PDPTaskAddRoot struct { -// db *harmonydb.DB -// sender *message.SenderETH -// ethClient *ethclient.Client -// filClient PDPServiceNodeApi -//} -// -//func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { -// ctx := context.Background() -// -// // Step 5: Prepare the Ethereum transaction data outside the DB transaction -// // Obtain the ABI of the PDPVerifier contract -// abiData, err := contract.PDPVerifierMetaData.GetAbi() -// if err != nil { -// return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) -// } -// -// // Prepare RootData array for Ethereum transaction -// // Define a Struct that matches the Solidity RootData struct -// type RootData struct { -// Root struct{ Data []byte } -// RawSize *big.Int -// } -// -// var rootDataArray []RootData -// -// rootData := RootData{ -// Root: struct{ Data []byte }{Data: rootCID.Bytes()}, -// RawSize: new(big.Int).SetUint64(totalSize), -// } -// -// // Step 6: Prepare the Ethereum transaction -// // Pack the method call data -// // The extraDataBytes variable is now correctly populated above -// data, err := abiData.Pack("addRoots", proofSetID, rootDataArray, extraDataBytes) -// if err != nil { -// return false, xerrors.Errorf("packing data: %w", err) -// } -// -// // Step 7: Get the sender address from 'eth_keys' table where role = 'pdp' limit 1 -// fromAddress, err := p.getSenderAddress(ctx) -// if err != nil { -// return false, xerrors.Errorf("getting sender address: %w", err) -// } -// -// // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) -// txEth := types.NewTransaction( -// 0, -// contract.ContractAddresses().PDPVerifier, -// big.NewInt(0), -// 0, -// nil, -// data, -// ) -// -// // Step 8: Send the transaction using SenderETH -// reason := "pdp-addroots" -// txHash, err := p.sender.Send(ctx, fromAddress, txEth, reason) -// if err != nil { -// return false, xerrors.Errorf("sending transaction: %w", err) -// } -// -// // Step 9: Insert into message_waits_eth and pdp_proofset_roots -// _, err = p.db.BeginTransaction(ctx, func(txdb *harmonydb.Tx) (bool, error) { -// // Insert into message_waits_eth -// _, err = txdb.Exec(` -// INSERT INTO message_waits_eth (signed_tx_hash, tx_status) -// VALUES ($1, $2) -// `, txHash.Hex(), "pending") -// if err != nil { -// return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) -// } -// -// // Update proof set for initialization upon first add -// _, err = txdb.Exec(` -// UPDATE pdp_proof_sets SET init_ready = true -// WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL -// `, proofSetIDUint64) -// if err != nil { -// return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) -// } -// -// // Insert into pdp_proofset_roots -// -// for addMessageIndex, addRootReq := range payload.Roots { -// for _, subrootEntry := range addRootReq.Subroots { -// subrootInfo := subrootInfoMap[subrootEntry.SubrootCID] -// -// // Insert into pdp_proofset_roots -// _, err = txdb.Exec(` -// INSERT INTO pdp_proofset_root_adds ( -// proofset, -// root, -// add_message_hash, -// add_message_index, -// subroot, -// subroot_offset, -// subroot_size, -// pdp_pieceref -// ) -// VALUES ($1, $2, $3, $4, $5, $6, $7, $8) -// `, -// proofSetIDUint64, -// addRootReq.RootCID, -// txHash.Hex(), -// addMessageIndex, -// subrootEntry.SubrootCID, -// subrootInfo.SubrootOffset, -// subrootInfo.PieceInfo.Size, -// subrootInfo.PDPPieceRefID, -// ) -// if err != nil { -// return false, err -// } -// } -// } -// -// // Return true to commit the transaction -// return true, nil -// }, harmonydb.OptionRetry()) -// if err != nil { -// return false, xerrors.Errorf("failed to save details to DB: %w", err) -// } -// return true, nil -//} -// -//// getSenderAddress retrieves the sender address from the database where role = 'pdp' limit 1 -//func (p *PDPTaskAddRoot) getSenderAddress(ctx context.Context) (common.Address, error) { -// var addressStr string -// err := p.db.QueryRow(ctx, `SELECT address FROM eth_keys WHERE role = 'pdp' LIMIT 1`).Scan(&addressStr) -// if err != nil { -// if errors.Is(err, sql.ErrNoRows) { -// return common.Address{}, errors.New("no sender address with role 'pdp' found") -// } -// return common.Address{}, err -// } -// address := common.HexToAddress(addressStr) -// return address, nil -//} -// -//func (p *PDPTaskAddRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { -// return &ids[0], nil -//} -// -//func (p *PDPTaskAddRoot) TypeDetails() harmonytask.TaskTypeDetails { -// return harmonytask.TaskTypeDetails{ -// Max: taskhelp.Max(50), -// Name: "PDPAddRoot", -// Cost: resources.Resources{ -// Cpu: 1, -// Ram: 64 << 20, -// }, -// MaxFailures: 3, -// IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { -// return p.schedule(context.Background(), taskFunc) -// }), -// } -//} -// -//func (p *PDPTaskAddRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} -// -//var _ harmonytask.TaskInterface = &PDPTaskAddRoot{} +import ( + "context" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/curio/pdp/contract" + "github.com/filecoin-project/curio/tasks/message" + + types2 "github.com/filecoin-project/lotus/chain/types" +) + +type PDPServiceNodeApi interface { + ChainHead(ctx context.Context) (*types2.TipSet, error) +} + +type PDPTaskAddRoot struct { + db *harmonydb.DB + sender *message.SenderETH + ethClient *ethclient.Client +} + +func NewPDPTaskAddRoot(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskAddRoot { + return &PDPTaskAddRoot{ + db: db, + sender: sender, + ethClient: ethClient, + } +} + +func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var addRoots []struct { + ID string `db:"id"` + PieceCid string `db:"piece_cid"` + ProofSetID int64 `db:"proof_set_id"` + ExtraData []byte `db:"extra_data"` + PieceRef string `db:"piece_ref"` + } + + err = p.db.Select(ctx, &addRoots, `SELECT id, piece_cid, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_root_task_id = $1 AND after_add_root = FALSE`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to select addRoot: %w", err) + } + + if len(addRoots) == 0 { + return false, xerrors.Errorf("no addRoot found for taskID %d", taskID) + } + + if len(addRoots) > 0 { + return false, xerrors.Errorf("multiple addRoot found for taskID %d", taskID) + } + + addRoot := addRoots[0] + + pcid, err := cid.Parse(addRoot.PieceCid) + if err != nil { + return false, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pi, err := mk20.GetPieceInfo(pcid) + if err != nil { + return false, xerrors.Errorf("failed to get piece info: %w", err) + } + + // Prepare the Ethereum transaction data outside the DB transaction + // Obtain the ABI of the PDPVerifier contract + abiData, err := contract.PDPVerifierMetaData.GetAbi() + if err != nil { + return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) + } + + rootDataArray := []contract.RootData{ + { + Root: struct{ Data []byte }{Data: pcid.Bytes()}, + RawSize: new(big.Int).SetUint64(pi.RawSize), + }, + } + + proofSetID := new(big.Int).SetUint64(uint64(addRoot.ProofSetID)) + + // Prepare the Ethereum transaction + // Pack the method call data + // The extraDataBytes variable is now correctly populated above + data, err := abiData.Pack("addRoots", proofSetID, rootDataArray, addRoot.ExtraData) + if err != nil { + return false, xerrors.Errorf("packing data: %w", err) + } + + callOpts := &bind.CallOpts{ + Context: ctx, + } + + pdpContracts := contract.ContractAddresses() + pdpVerifierAddress := pdpContracts.PDPVerifier + + pdpVerifier, err := contract.NewPDPVerifier(pdpVerifierAddress, p.ethClient) + if err != nil { + return false, xerrors.Errorf("failed to instantiate PDPVerifier contract at %s: %w", pdpVerifierAddress.Hex(), err) + } + + // Get the sender address for this proofset + owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + if err != nil { + return false, xerrors.Errorf("failed to get owner: %w", err) + } + + // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) + txEth := types.NewTransaction( + 0, + contract.ContractAddresses().PDPVerifier, + big.NewInt(0), + 0, + nil, + data, + ) + + // Send the transaction using SenderETH + reason := "pdp-addroots" + txHash, err := p.sender.Send(ctx, owner, txEth, reason) + if err != nil { + return false, xerrors.Errorf("sending transaction: %w", err) + } + + // Insert into message_waits_eth and pdp_proofset_roots + _, err = p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + // Insert into message_waits_eth + _, err = tx.Exec(` + INSERT INTO message_waits_eth (signed_tx_hash, tx_status) + VALUES ($1, $2) + `, txHash.Hex(), "pending") + if err != nil { + return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) + } + + // Update proof set for initialization upon first add + _, err = tx.Exec(` + UPDATE pdp_proof_sets SET init_ready = true + WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL + `, proofSetID.Uint64()) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + } + + // Insert into pdp_proofset_roots + n, err := tx.Exec(` + INSERT INTO pdp_proofset_root ( + proofset, + piece_cid_v2, + piece_cid, + piece_size, + raw_size, + piece_ref, + add_deal_id, + add_message_hash + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + `, + proofSetID.Uint64(), + pcid.String(), + pi.PieceCIDV1.String(), + pi.Size, + pi.RawSize, + addRoot.PieceRef, + addRoot.ID, + txHash.Hex(), + ) + if err != nil { + return false, xerrors.Errorf("failed to insert into pdp_proofset_root: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows inserted for pdp_proofset_root: %d", n) + } + + n, err = tx.Exec(`UPDATE pdp_pipeline SET + after_add_root = TRUE, + add_root_task_id = NULL, + add_message_hash = $2, + WHERE add_root_task_id = $1`, taskID, txHash.Hex()) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_pipeline: %d", n) + } + + // Return true to commit the transaction + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("failed to save details to DB: %w", err) + } + return true, nil +} + +func (p *PDPTaskAddRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (p *PDPTaskAddRoot) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "PDPAddRoot", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return p.schedule(context.Background(), taskFunc) + }), + } +} + +func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_pipeline + WHERE add_root_task_id IS NULL + AND after_add_root = FALSE + AND after_add_root_msg = FALSE + AND aggregated = TRUE`).Scan(&did) + if err != nil { + return false, xerrors.Errorf("failed to query pdp_pipeline: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid deal ID found for scheduling") + } + + _, err = tx.Exec(`UPDATE pdp_pipeline SET add_root_task_id = $1, WHERE piece_cid = $2 AND after_add_root = FALSE AND after_add_root_msg = FALSE AND aggregated = TRUE`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (p *PDPTaskAddRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &PDPTaskAddRoot{} diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go new file mode 100644 index 000000000..6c5012c89 --- /dev/null +++ b/tasks/pdp/task_aggregation.go @@ -0,0 +1,353 @@ +package pdp + +import ( + "context" + "fmt" + "io" + "math/bits" + "time" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-data-segment/datasegment" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" +) + +type AggregatePDPDealTask struct { + db *harmonydb.DB + sc *ffi.SealCalls +} + +func NewAggregatePDPDealTask(db *harmonydb.DB, sc *ffi.SealCalls) *AggregatePDPDealTask { + return &AggregatePDPDealTask{ + db: db, + sc: sc, + } +} + +func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var pieces []struct { + Pcid string `db:"piece_cid"` + Psize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + PieceRef int64 `db:"piece_ref"` + ID string `db:"id"` + AggrIndex int `db:"aggr_index"` + Aggregated bool `db:"aggregated"` + Aggregation int `db:"deal_aggregation"` + } + + err = a.db.Select(ctx, &pieces, ` + SELECT + piece_cid, + piece_size, + raw_size, + piece_ref, + id, + aggr_index, + aggregated, + deal_aggregation + FROM + pdp_pipeline + WHERE + agg_task_id = $1 ORDER BY aggr_index ASC`, taskID) + if err != nil { + return false, xerrors.Errorf("getting piece details: %w", err) + } + + if len(pieces) == 0 { + return false, xerrors.Errorf("no pieces to aggregate for task %d", taskID) + } + + if len(pieces) == 1 { + n, err := a.db.Exec(ctx, `UPDATE pdp_pipeline SET aggregated = TRUE, agg_task_id = NULL + WHERE id = $1 + AND agg_task_id = $2`, pieces[0].ID, taskID) + if err != nil { + return false, xerrors.Errorf("updating aggregated piece details in DB: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row updated, got %d", n) + } + log.Infof("skipping aggregation as deal %s only has 1 piece for task %s", pieces[0].ID, taskID) + return true, nil + } + + id := pieces[0].ID + + ID, err := ulid.Parse(id) + if err != nil { + return false, xerrors.Errorf("parsing deal ID: %w", err) + } + + deal, err := mk20.DealFromDB(ctx, a.db, ID) + if err != nil { + return false, xerrors.Errorf("getting deal details from DB: %w", err) + } + + pi, err := deal.PieceInfo() + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + + var pinfos []abi.PieceInfo + var readers []io.Reader + + var refIDs []int64 + + for _, piece := range pieces { + if piece.Aggregated { + return false, xerrors.Errorf("piece %s for deal %s already aggregated for task %d", piece.Pcid, piece.ID, taskID) + } + if piece.Aggregation != 1 { + return false, xerrors.Errorf("incorrect aggregation value for piece %s for deal %s for task %d", piece.Pcid, piece.ID, taskID) + } + if piece.ID != id { + return false, xerrors.Errorf("piece details do not match") + } + + var reader io.Reader // io.ReadCloser is not supported by padreader + var closer io.Closer + + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = a.db.Select(ctx, &pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, piece.PieceRef) + if err != nil { + return false, xerrors.Errorf("getting pieceID: %w", err) + } + + if len(pieceID) != 1 { + return false, xerrors.Errorf("expected 1 pieceID, got %d", len(pieceID)) + } + + pr, err := a.sc.PieceReader(ctx, pieceID[0].PieceID) + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + + closer = pr + reader = pr + defer func() { + _ = closer.Close() + }() + + pcid, err := cid.Parse(piece.Pcid) + if err != nil { + return false, xerrors.Errorf("parsing piece cid: %w", err) + } + + pinfos = append(pinfos, abi.PieceInfo{ + Size: abi.PaddedPieceSize(piece.Psize), + PieceCID: pcid, + }) + + readers = append(readers, io.LimitReader(reader, piece.RawSize)) + refIDs = append(refIDs, piece.PieceRef) + } + + _, aggregatedRawSize, err := datasegment.ComputeDealPlacement(pinfos) + if err != nil { + return false, xerrors.Errorf("computing aggregated piece size: %w", err) + } + + overallSize := abi.PaddedPieceSize(aggregatedRawSize) + // we need to make this the 'next' power of 2 in order to have space for the index + next := 1 << (64 - bits.LeadingZeros64(uint64(overallSize+256))) + + aggr, err := datasegment.NewAggregate(abi.PaddedPieceSize(next), pinfos) + if err != nil { + return false, xerrors.Errorf("creating aggregate: %w", err) + } + + outR, err := aggr.AggregateObjectReader(readers) + if err != nil { + return false, xerrors.Errorf("aggregating piece readers: %w", err) + } + + var parkedPieceID, pieceRefID int64 + var pieceParked bool + + comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, + pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID, &pieceParked) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } + + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, "/Aggregate").Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("saving aggregated chunk details to DB: %w", err) + } + + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") + } + + failed := true + + // Clean up piece park tables in case of failure + // TODO: Figure out if there is a race condition with cleanup task + defer func() { + if failed { + _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) + if err != nil { + log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) + } + } + }() + + // Write piece if not already complete + if !pieceParked { + upi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage) + if err != nil { + return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) + } + + if !upi.PieceCID.Equals(pi.PieceCIDV1) { + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", upi.PieceCID.String(), pi.PieceCIDV1.String()) + } + + if upi.Size != pi.Size { + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", upi.Size, pi.Size) + } + } + + comm, err = a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // Replace the pipeline piece with a new aggregated piece + _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, id) + if err != nil { + return false, fmt.Errorf("failed to delete pipeline pieces: %w", err) + } + + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1) AND long_term = FALSE`, refIDs) + if err != nil { + return false, fmt.Errorf("failed to delete parked_piece_refs entries: %w", err) + } + + pdp := deal.Products.PDPV1 + + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting aggregated piece in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting aggregated piece in PDP pipeline: %d rows affected", n) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("saving aggregated piece details to DB: %w", err) + } + + if !comm { + return false, xerrors.Errorf("failed to commit the transaction") + } + + failed = false + + return true, nil +} + +func (a *AggregatePDPDealTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + // If no local pieceRef was found then just return first TaskID + return &ids[0], nil +} + +func (a *AggregatePDPDealTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "AggregatePDPDeal", + Cost: resources.Resources{ + Cpu: 1, + Ram: 4 << 30, + }, + MaxFailures: 3, + IAmBored: passcall.Every(3*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return a.schedule(context.Background(), taskFunc) + }), + } +} + +func (a *AggregatePDPDealTask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var deals []struct { + ID string `db:"id"` + Count int `db:"count"` + } + + err := a.db.Select(ctx, &deals, `SELECT id, COUNT(*) AS count + FROM pdp_pipeline + GROUP BY id + HAVING bool_and(downloaded) + AND bool_and(NOT aggregated) + AND bool_and(agg_task_id IS NULL);`) + if err != nil { + log.Errorf("getting deals to aggregate: %w", err) + return + } + + deal := deals[0] + + log.Infow("processing aggregation task", "deal", deal.ID, "count", deal.Count) + n, err := tx.Exec(`UPDATE pdp_pipeline SET agg_task_id = $1 + WHERE id = $2 + AND downloaded = TRUE + AND aggregated = FALSE + AND agg_task_id IS NULL`, id, deal.ID) + if err != nil { + return false, xerrors.Errorf("creating aggregation task for PDP: %w", err) + } + + if n == deal.Count { + log.Infow("aggregation task created successfully", "deal", deal.ID) + } + + stop = false + + return n == deal.Count, nil + }) + + } + + return nil +} + +func (a *AggregatePDPDealTask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ = harmonytask.Reg(&AggregatePDPDealTask{}) +var _ harmonytask.TaskInterface = &AggregatePDPDealTask{} diff --git a/tasks/pdp/task_init_pp.go b/tasks/pdp/task_init_pp.go index b5bce2010..8b637b900 100644 --- a/tasks/pdp/task_init_pp.go +++ b/tasks/pdp/task_init_pp.go @@ -55,7 +55,7 @@ func NewInitProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil err := db.Select(ctx, &toCallInit, ` SELECT id - FROM pdp_proof_sets + FROM pdp_proof_set WHERE challenge_request_task_id IS NULL AND init_ready AND prove_at_epoch IS NULL `) @@ -65,14 +65,14 @@ func NewInitProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil for _, ps := range toCallInit { ipp.addFunc.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { - // Update pdp_proof_sets to set challenge_request_task_id = id + // Update pdp_proof_set to set challenge_request_task_id = id affected, err := tx.Exec(` - UPDATE pdp_proof_sets + UPDATE pdp_proof_set SET challenge_request_task_id = $1 WHERE id = $2 AND challenge_request_task_id IS NULL `, id, ps.ProofSetID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_proof_set: %w", err) } if affected == 0 { // Someone else might have already scheduled the task @@ -97,7 +97,7 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( err = ipp.db.QueryRow(ctx, ` SELECT id - FROM pdp_proof_sets + FROM pdp_proof_set WHERE challenge_request_task_id = $1 `, taskID).Scan(&proofSetID) if err == sql.ErrNoRows { @@ -188,7 +188,7 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( _, err = ipp.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Update pdp_proof_sets affected, err := tx.Exec(` - UPDATE pdp_proof_sets + UPDATE pdp_proof_set SET challenge_request_msg_hash = $1, prev_challenge_request_epoch = $2, prove_at_epoch = $3 diff --git a/tasks/pdp/task_next_pp.go b/tasks/pdp/task_next_pp.go index 14790e844..29cde60a5 100644 --- a/tasks/pdp/task_next_pp.go +++ b/tasks/pdp/task_next_pp.go @@ -54,7 +54,7 @@ func NewNextProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil err := db.Select(ctx, &toCallNext, ` SELECT id - FROM pdp_proof_sets + FROM pdp_proof_set WHERE challenge_request_task_id IS NULL AND (prove_at_epoch + challenge_window) <= $1 `, apply.Height()) @@ -66,7 +66,7 @@ func NewNextProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil n.addFunc.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { // Update pdp_proof_sets to set challenge_request_task_id = id affected, err := tx.Exec(` - UPDATE pdp_proof_sets + UPDATE pdp_proof_set SET challenge_request_task_id = $1 WHERE id = $2 AND challenge_request_task_id IS NULL `, id, ps.ProofSetID) @@ -95,7 +95,7 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() err = n.db.QueryRow(ctx, ` SELECT id - FROM pdp_proof_sets + FROM pdp_proof_set WHERE challenge_request_task_id = $1 AND prove_at_epoch IS NOT NULL `, taskID).Scan(&proofSetID) if err == sql.ErrNoRows { @@ -179,7 +179,7 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() _, err = n.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Update pdp_proof_sets affected, err := tx.Exec(` - UPDATE pdp_proof_sets + UPDATE pdp_proof_set SET challenge_request_msg_hash = $1, prev_challenge_request_epoch = $2, prove_at_epoch = $3 diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index 3e2daab88..0cb9958a6 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -8,8 +8,6 @@ import ( "errors" "io" "math/big" - "math/bits" - "sort" "sync/atomic" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -17,14 +15,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ipfs/go-cid" - pool "github.com/libp2p/go-buffer-pool" "github.com/minio/sha256-simd" "github.com/samber/lo" "golang.org/x/crypto/sha3" "golang.org/x/xerrors" - "github.com/filecoin-project/go-commp-utils/zerocomm" - commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -35,6 +31,8 @@ import ( "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/lib/promise" "github.com/filecoin-project/curio/lib/proof" + "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/pdp/contract" "github.com/filecoin-project/curio/tasks/message" @@ -50,6 +48,7 @@ type ProveTask struct { sender *message.SenderETH cpr *cachedreader.CachedPieceReader fil ProveTaskChainApi + idx *indexstore.IndexStore head atomic.Pointer[chainTypes.TipSet] @@ -61,13 +60,14 @@ type ProveTaskChainApi interface { ChainHead(context.Context) (*chainTypes.TipSet, error) //perm:read } -func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethClient *ethclient.Client, fil ProveTaskChainApi, sender *message.SenderETH, cpr *cachedreader.CachedPieceReader) *ProveTask { +func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethClient *ethclient.Client, fil ProveTaskChainApi, sender *message.SenderETH, cpr *cachedreader.CachedPieceReader, idx *indexstore.IndexStore) *ProveTask { pt := &ProveTask{ db: db, ethClient: ethClient, sender: sender, cpr: cpr, fil: fil, + idx: idx, } // ProveTasks are created on pdp_proof_sets entries where @@ -91,7 +91,7 @@ func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethC err := tx.Select(&proofSets, ` SELECT p.id - FROM pdp_proof_sets p + FROM pdp_proof_set p INNER JOIN message_waits_eth mw on mw.signed_tx_hash = p.challenge_request_msg_hash WHERE p.challenge_request_msg_hash IS NOT NULL AND mw.tx_success = TRUE AND p.prove_at_epoch < $1 LIMIT 2 @@ -125,7 +125,7 @@ func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethC // Update pdp_proof_sets to set next_challenge_possible = FALSE affected, err = tx.Exec(` - UPDATE pdp_proof_sets + UPDATE pdp_proof_set SET challenge_request_msg_hash = NULL WHERE id = $1 AND challenge_request_msg_hash IS NOT NULL `, todo.ID) @@ -383,263 +383,244 @@ func padTo32Bytes(b []byte) []byte { return padded } -func (p *ProveTask) genSubrootMemtree(ctx context.Context, subrootCid string, subrootSize abi.PaddedPieceSize) ([]byte, error) { - subrootCidObj, err := cid.Parse(subrootCid) - if err != nil { - return nil, xerrors.Errorf("failed to parse subroot CID: %w", err) - } - +func (p *ProveTask) genSubrootMemtree( + ctx context.Context, + pieceCidV2 cid.Cid, + challengedLeafIndex int64, + savedLayer int, +) ([]byte, error) { + // Calculate which snapshot node covers this challenged leaf + leavesPerNode := int64(1) << savedLayer + snapshotNodeIndex := challengedLeafIndex >> savedLayer + startLeaf := snapshotNodeIndex << savedLayer + + // Convert tree-based leaf range to file-based offset/length + offset := startLeaf * inputBytesPerLeaf + length := leavesPerNode * inputBytesPerLeaf + + // Compute padded size to build Merkle tree (must match what BuildSha254Memtree expects) + subrootSize := padreader.PaddedSize(uint64(length)).Padded() if subrootSize > proof.MaxMemtreeSize { return nil, xerrors.Errorf("subroot size exceeds maximum: %d", subrootSize) } - commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{PieceCID: subrootCidObj, Size: subrootSize}) + // Get original file reader + reader, reportedSize, err := p.cpr.GetSharedPieceReader(ctx, pieceCidV2) if err != nil { - return nil, xerrors.Errorf("failed to get piece commitment: %w", err) + return nil, xerrors.Errorf("failed to get reader: %w", err) + } + defer reader.Close() + + if offset > int64(reportedSize) { + // The entire requested range is beyond file size ā†’ pure padding + // This should never happen + //TODO: Maybe put a panic here? + paddingOnly := nullreader.NewNullReader(abi.UnpaddedPieceSize(length)) + return proof.BuildSha254Memtree(paddingOnly, subrootSize.Unpadded()) } - subrootReader, unssize, err := p.cpr.GetSharedPieceReader(ctx, commp.PCidV2()) + _, err = reader.Seek(offset, io.SeekStart) if err != nil { - return nil, xerrors.Errorf("failed to get subroot reader: %w", err) + return nil, xerrors.Errorf("seek to offset %d failed: %w", offset, err) } - var r io.Reader = subrootReader + // Read up to file limit + var data io.Reader + fileRemaining := int64(reportedSize) - offset + if fileRemaining < length { + data = io.MultiReader(io.LimitReader(reader, fileRemaining), nullreader.NewNullReader(abi.UnpaddedPieceSize(int64(subrootSize.Unpadded())-fileRemaining))) + } else { + data = io.LimitReader(reader, length) + } + + // Build Merkle tree from padded input + return proof.BuildSha254Memtree(data, subrootSize.Unpadded()) +} - if unssize.Padded() > subrootSize { - return nil, xerrors.Errorf("subroot size mismatch: %d > %d", unssize.Padded(), subrootSize) - } else if unssize.Padded() < subrootSize { - // pad with zeros - r = io.MultiReader(r, nullreader.NewNullReader(abi.UnpaddedPieceSize(subrootSize-unssize.Padded()))) +func GenerateProofToRootFromSnapshot( + snapshotLayer int, + snapshotIndex int64, + snapshotHash [32]byte, + snapshotNodes []indexstore.NodeDigest, +) ([][32]byte, [32]byte, error) { + snapMap := make(map[int64][32]byte) + for _, n := range snapshotNodes { + if n.Layer != snapshotLayer { + continue // ignore other layers if present + } + snapMap[n.Index] = n.Hash } - defer subrootReader.Close() + proof := make([][32]byte, 0) + currentHash := snapshotHash + currentIndex := snapshotIndex + hasher := sha256.New() + + for level := snapshotLayer + 1; ; level++ { + siblingIndex := currentIndex ^ 1 + + siblingHash, exists := snapMap[siblingIndex] + if !exists { + // Padding if sibling missing + siblingHash = currentHash + } + + // Add sibling to proof + proof = append(proof, siblingHash) + + // Compute parent + hasher.Reset() + if currentIndex%2 == 0 { + hasher.Write(currentHash[:]) + hasher.Write(siblingHash[:]) + } else { + hasher.Write(siblingHash[:]) + hasher.Write(currentHash[:]) + } + sum := hasher.Sum(nil) + var parent [32]byte + copy(parent[:], sum) + parent[31] &= 0x3F + + currentHash = parent + currentIndex = currentIndex >> 1 + + // stop when we reach the root (single node at level) + if len(snapMap) <= 1 && currentIndex == 0 { + break + } + } - return proof.BuildSha254Memtree(r, subrootSize.Unpadded()) + return proof, currentHash, nil } func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int64, challengedLeaf int64) (contract.PDPVerifierProof, error) { - const arity = 2 + //const arity = 2 rootChallengeOffset := challengedLeaf * LeafSize - // Retrieve the root and subroot - type subrootMeta struct { - Root string `db:"root"` - Subroot string `db:"subroot"` - SubrootOffset int64 `db:"subroot_offset"` // padded offset - SubrootSize int64 `db:"subroot_size"` // padded piece size - } - - var subroots []subrootMeta + var pieceCid string - err := p.db.Select(context.Background(), &subroots, ` - SELECT root, subroot, subroot_offset, subroot_size - FROM pdp_proofset_roots - WHERE proofset = $1 AND root_id = $2 - ORDER BY subroot_offset ASC - `, proofSetID, rootId) + err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_proofset_root WHERE proofset = $1 AND root_id = $2`, proofSetID, rootId).Scan(&pieceCid) if err != nil { return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get root and subroot: %w", err) } - // find first subroot with subroot_offset >= rootChallengeOffset - challSubRoot, challSubrootIdx, ok := lo.FindLastIndexOf(subroots, func(subroot subrootMeta) bool { - return subroot.SubrootOffset < rootChallengeOffset - }) - if !ok { - return contract.PDPVerifierProof{}, xerrors.New("no subroot found") - } - - // build subroot memtree - memtree, err := p.genSubrootMemtree(ctx, challSubRoot.Subroot, abi.PaddedPieceSize(challSubRoot.SubrootSize)) + pcid, err := cid.Parse(pieceCid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate subroot memtree: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to parse piece CID: %w", err) } - subrootChallengedLeaf := challengedLeaf - (challSubRoot.SubrootOffset / LeafSize) - log.Debugw("subrootChallengedLeaf", "subrootChallengedLeaf", subrootChallengedLeaf, "challengedLeaf", challengedLeaf, "subrootOffsetLs", challSubRoot.SubrootOffset/LeafSize) - - /* - type RawMerkleProof struct { - Leaf [32]byte - Proof [][32]byte - Root [32]byte - } - */ - subrootProof, err := proof.MemtreeProof(memtree, subrootChallengedLeaf) - pool.Put(memtree) + pi, err := mk20.GetPieceInfo(pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate subroot proof: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece info: %w", err) } - log.Debugw("subrootProof", "subrootProof", subrootProof) - // build partial top-tree - type treeElem struct { - Level int // 1 == leaf, NODE_SIZE - Hash [LeafSize]byte - } - type elemIndex struct { - Level int - ElemOffset int64 // offset in terms of nodes at the current level - } - - partialTree := map[elemIndex]treeElem{} - var subrootsSize abi.PaddedPieceSize - - // 1. prefill the partial tree - for _, subroot := range subroots { - subrootsSize += abi.PaddedPieceSize(subroot.SubrootSize) + var out contract.PDPVerifierProof + var rootDigest [32]byte - unsCid, err := cid.Parse(subroot.Subroot) + // If piece is less than 100 MiB, let's generate proof directly without using cache + if pi.RawSize < MinSizeForCache { + // Get original file reader + reader, _, err := p.cpr.GetSharedPieceReader(ctx, pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to parse subroot CID: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece reader: %w", err) } + defer reader.Close() - commp, err := commcid.CIDToPieceCommitmentV1(unsCid) + // Build Merkle tree from padded input + memTree, err := proof.BuildSha254Memtree(reader, pi.Size.Unpadded()) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to convert CID to piece commitment: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree: %w", err) } + log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf) - var comm [LeafSize]byte - copy(comm[:], commp) - - level := proof.NodeLevel(subroot.SubrootSize/LeafSize, arity) - offset := (subroot.SubrootOffset / LeafSize) >> uint(level-1) - partialTree[elemIndex{Level: level, ElemOffset: offset}] = treeElem{ - Level: level, - Hash: comm, + mProof, err := proof.MemtreeProof(memTree, challengedLeaf) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) } - } - - rootSize := nextPowerOfTwo(subrootsSize) - rootLevel := proof.NodeLevel(int64(rootSize/LeafSize), arity) - // 2. build the partial tree - // we do the build from the right side of the tree - elements are sorted by size, so only elements on the right side can have missing siblings - - isRight := func(offset int64) bool { - return offset&1 == 1 - } + out = contract.PDPVerifierProof{ + Leaf: mProof.Leaf, + Proof: mProof.Proof, + } - for i := len(subroots) - 1; i >= 0; i-- { - subroot := subroots[i] - level := proof.NodeLevel(subroot.SubrootSize/LeafSize, arity) - offset := (subroot.SubrootOffset / LeafSize) >> uint(level-1) - firstSubroot := i == 0 + rootDigest = mProof.Root + } else { + layerIdx := snapshotLayerIndex(pi.RawSize) + cacheIdx := challengedLeaf >> layerIdx - curElem := partialTree[elemIndex{Level: level, ElemOffset: offset}] + has, node, err := p.idx.GetPDPNode(ctx, pcid, cacheIdx) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get node: %w", err) + } - log.Debugw("processing partialtree subroot", "curElem", curElem, "level", level, "offset", offset, "subroot", subroot.SubrootOffset, "subrootSz", subroot.SubrootSize) + if !has { + // TODO: Trigger a Layer save task here and figure out if we should proceed or not + // TODO: Proceeding from here can cause memory issue for big pieces, we will need to generate proof using some other lib + panic("implement me") + } - for !isRight(offset) { - // find the rightSibling - siblingIndex := elemIndex{Level: level, ElemOffset: offset + 1} - rightSibling, ok := partialTree[siblingIndex] - if !ok { - // if we're processing the first subroot branch, AND we've ran out of right siblings, we're done - if firstSubroot { - break - } + log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf, "layerIdx", layerIdx, "cacheIdx", cacheIdx, "node", node) - // create a zero rightSibling - rightSibling = treeElem{ - Level: level, - Hash: zerocomm.PieceComms[level-zerocomm.Skip-1], - } - log.Debugw("rightSibling zero", "rightSibling", rightSibling, "siblingIndex", siblingIndex, "level", level, "offset", offset) - partialTree[siblingIndex] = rightSibling - } + if node.Layer != layerIdx { + return contract.PDPVerifierProof{}, xerrors.Errorf("node layer mismatch: %d != %d", node.Layer, layerIdx) + } - // compute the parent - parent := proof.ComputeBinShaParent(curElem.Hash, rightSibling.Hash) - parentLevel := level + 1 - parentOffset := offset / arity + // build subroot memtree + memtree, err := p.genSubrootMemtree(ctx, pcid, challengedLeaf, layerIdx) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate subroot memtree: %w", err) + } - partialTree[elemIndex{Level: parentLevel, ElemOffset: parentOffset}] = treeElem{ - Level: parentLevel, - Hash: parent, + /* + type RawMerkleProof struct { + Leaf [32]byte + Proof [][32]byte + Root [32]byte } - - // move to the parent - level = parentLevel - offset = parentOffset - curElem = partialTree[elemIndex{Level: level, ElemOffset: offset}] + */ + subTreeProof, err := proof.MemtreeProof(memtree, challengedLeaf) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate sub tree proof: %w", err) } - } + log.Debugw("subTreeProof", "subrootProof", subTreeProof) - { - var partialTreeList []elemIndex - for k := range partialTree { - partialTreeList = append(partialTreeList, k) + // Verify root of proof + if subTreeProof.Root != node.Hash { + return contract.PDPVerifierProof{}, xerrors.Errorf("subroot root mismatch: %x != %x", subTreeProof.Root, node.Hash) } - sort.Slice(partialTreeList, func(i, j int) bool { - if partialTreeList[i].Level != partialTreeList[j].Level { - return partialTreeList[i].Level < partialTreeList[j].Level - } - return partialTreeList[i].ElemOffset < partialTreeList[j].ElemOffset - }) - - } - - challLevel := proof.NodeLevel(challSubRoot.SubrootSize/LeafSize, arity) - challOffset := (challSubRoot.SubrootOffset / LeafSize) >> uint(challLevel-1) - - log.Debugw("challSubRoot", "challSubRoot", challSubrootIdx, "challLevel", challLevel, "challOffset", challOffset) - - challSubtreeLeaf := partialTree[elemIndex{Level: challLevel, ElemOffset: challOffset}] - if challSubtreeLeaf.Hash != subrootProof.Root { - return contract.PDPVerifierProof{}, xerrors.Errorf("subtree root doesn't match partial tree leaf, %x != %x", challSubtreeLeaf.Hash, subrootProof.Root) - } - var out contract.PDPVerifierProof - copy(out.Leaf[:], subrootProof.Leaf[:]) - out.Proof = append(out.Proof, subrootProof.Proof...) - - currentLevel := challLevel - currentOffset := challOffset - - for currentLevel < rootLevel { - siblingOffset := currentOffset ^ 1 - - // Retrieve sibling hash from partialTree or use zero hash - siblingIndex := elemIndex{Level: currentLevel, ElemOffset: siblingOffset} - index := elemIndex{Level: currentLevel, ElemOffset: currentOffset} - siblingElem, ok := partialTree[siblingIndex] - if !ok { - return contract.PDPVerifierProof{}, xerrors.Errorf("missing sibling at level %d, offset %d", currentLevel, siblingOffset) - } - elem, ok := partialTree[index] - if !ok { - return contract.PDPVerifierProof{}, xerrors.Errorf("missing element at level %d, offset %d", currentLevel, currentOffset) + // Fetch full cached layer from DB + layerNodes, err := p.idx.GetPDPLayer(ctx, pcid) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get layer nodes: %w", err) } - if currentOffset < siblingOffset { // left - log.Debugw("Proof", "position", index, "left-c", hex.EncodeToString(elem.Hash[:]), "right-s", hex.EncodeToString(siblingElem.Hash[:]), "out", hex.EncodeToString(shabytes(append(elem.Hash[:], siblingElem.Hash[:]...))[:])) - } else { // right - log.Debugw("Proof", "position", index, "left-s", hex.EncodeToString(siblingElem.Hash[:]), "right-c", hex.EncodeToString(elem.Hash[:]), "out", hex.EncodeToString(shabytes(append(siblingElem.Hash[:], elem.Hash[:]...))[:])) + + proofs, rd, err := GenerateProofToRootFromSnapshot(node.Layer, node.Index, node.Hash, layerNodes) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate proof to root: %w", err) } - // Append the sibling's hash to the proof - out.Proof = append(out.Proof, siblingElem.Hash) + com, err := commcidv2.CommPFromPCidV2(pcid) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece commitment: %w", err) + } - // Move up to the parent node - currentOffset = currentOffset / arity - currentLevel++ - } + // Verify proof with original root + if [32]byte(com.Digest()) != rd { + return contract.PDPVerifierProof{}, xerrors.Errorf("root digest mismatch: %x != %x", com.Digest(), rd) + } - log.Debugw("proof complete", "proof", out) + out = contract.PDPVerifierProof{ + Leaf: subTreeProof.Leaf, + Proof: append([][32]byte{subTreeProof.Root}, proofs...), + } - rootCid, err := cid.Parse(subroots[0].Root) - if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to parse root CID: %w", err) + rootDigest = rd } - commRoot, err := commcid.CIDToPieceCommitmentV1(rootCid) - if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to convert CID to piece commitment: %w", err) - } - var cr [LeafSize]byte - copy(cr[:], commRoot) - if !Verify(out, cr, uint64(challengedLeaf)) { + if !Verify(out, rootDigest, uint64(challengedLeaf)) { return contract.PDPVerifierProof{}, xerrors.Errorf("proof verification failed") } @@ -743,11 +724,6 @@ func (p *ProveTask) Adder(taskFunc harmonytask.AddTaskFunc) { p.addFunc.Set(taskFunc) } -func nextPowerOfTwo(n abi.PaddedPieceSize) abi.PaddedPieceSize { - lz := bits.LeadingZeros64(uint64(n - 1)) - return 1 << (64 - lz) -} - func Verify(proof contract.PDPVerifierProof, root [32]byte, position uint64) bool { computedHash := proof.Leaf diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go new file mode 100644 index 000000000..63b2c91b7 --- /dev/null +++ b/tasks/pdp/task_save_cache.go @@ -0,0 +1,604 @@ +package pdp + +import ( + "context" + "hash" + "io" + "math/bits" + "sync" + "time" + + "github.com/ipfs/go-cid" + sha256simd "github.com/minio/sha256-simd" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/mk20" +) + +const MinSizeForCache = uint64(100 * 1024 * 1024) +const CacheReadSize = int64(4 * 1024 * 1024) + +type TaskSavePDPCache struct { + db *harmonydb.DB + cpr *cachedreader.CachedPieceReader + idx *indexstore.IndexStore +} + +func NewTaskSavePDPCache(db *harmonydb.DB, cpr *cachedreader.CachedPieceReader, idx *indexstore.IndexStore) *TaskSavePDPCache { + return &TaskSavePDPCache{ + db: db, + cpr: cpr, + idx: idx, + } +} + +func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + var saveCaches []struct { + ID string `db:"id"` + PieceCid string `db:"piece_cid"` + ProofSetID int64 `db:"proof_set_id"` + ExtraData []byte `db:"extra_data"` + PieceRef string `db:"piece_ref"` + } + + err = t.db.Select(ctx, &saveCaches, `SELECT id, piece_cid, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE save_cache_task_id = $1 AND after_save_cache = FALSE`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to select addRoot: %w", err) + } + + if len(saveCaches) == 0 { + return false, xerrors.Errorf("no saveCaches found for taskID %d", taskID) + } + + if len(saveCaches) > 0 { + return false, xerrors.Errorf("multiple saveCaches found for taskID %d", taskID) + } + + sc := saveCaches[0] + + pcid, err := cid.Parse(sc.PieceCid) + if err != nil { + return false, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pi, err := mk20.GetPieceInfo(pcid) + if err != nil { + return false, xerrors.Errorf("failed to get piece info: %w", err) + } + + // Let's build the merkle Tree again (commP) and save a middle layer for fast proving + // for pieces larger than 100 MiB + if pi.RawSize > MinSizeForCache { + has, err := t.idx.HasPDPLayer(ctx, pcid) + if err != nil { + return false, xerrors.Errorf("failed to check if piece has PDP layer: %w", err) + } + + if !has { + cp := NewCommPWithSize(pi.RawSize) + reader, _, err := t.cpr.GetSharedPieceReader(ctx, pcid) + if err != nil { + return false, xerrors.Errorf("failed to get shared piece reader: %w", err) + } + defer reader.Close() + + n, err := io.CopyBuffer(cp, reader, make([]byte, 4<<20)) + if err != nil { + return false, xerrors.Errorf("failed to copy piece data to commP: %w", err) + } + + digest, _, lidx, snap, err := cp.DigestWithSnapShot() + if err != nil { + return false, xerrors.Errorf("failed to get piece digest: %w", err) + } + + com, err := commcidv2.NewSha2CommP(uint64(n), digest) + if err != nil { + return false, xerrors.Errorf("failed to create commP: %w", err) + } + + if !com.PCidV2().Equals(pcid) { + return false, xerrors.Errorf("commP cid does not match piece cid: %s != %s", com.PCidV2().String(), pcid.String()) + } + + leafs := make([]indexstore.NodeDigest, len(snap)) + for i, s := range snap { + leafs[i] = indexstore.NodeDigest{ + Layer: lidx, + Hash: s.Hash, + Index: s.Index, + } + } + + err = t.idx.AddPDPLayer(ctx, pcid, leafs) + if err != nil { + return false, xerrors.Errorf("failed to add PDP layer cache: %w", err) + } + } + } + + n, err := t.db.Exec(ctx, `UPDATE pdp_pipeline SET after_save_cache = TRUE, save_cache_task_id = NULL WHERE save_cache_task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) + } + + if n != 1 { + return false, xerrors.Errorf("failed to update pdp_pipeline: expected 1 row but %d rows updated", n) + } + + return true, nil +} + +func (t *TaskSavePDPCache) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (t *TaskSavePDPCache) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "SavePDPCache", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(2*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return t.schedule(context.Background(), taskFunc) + }), + } +} + +func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_pipeline + WHERE save_cache_task_id IS NULL + AND save_cache_task_id = FALSE + AND aggregated = TRUE`).Scan(&did) + if err != nil { + return false, xerrors.Errorf("failed to query pdp_pipeline: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid deal ID found for scheduling") + } + + _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1, WHERE id = $2 AND save_cache_task_id = FALSE AND aggregated = TRUE`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (t *TaskSavePDPCache) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &TaskSavePDPCache{} + +// All the code below is a copy+paste of https://github.com/filecoin-project/go-fil-commp-hashhash/blob/master/commp.go +// with modification to output the nodes at a specific height + +// Calc is an implementation of a commP "hash" calculator, implementing the +// familiar hash.Hash interface. The zero-value of this object is ready to +// accept Write()s without further initialization. +type Calc struct { + state + mu sync.Mutex +} +type state struct { + quadsEnqueued uint64 + layerQueues [MaxLayers + 2]chan []byte // one extra layer for the initial leaves, one more for the dummy never-to-use channel + resultCommP chan []byte + buffer []byte + size uint64 + snapShotLayerIdx int + snapshotNodes []NodeDigest + snapshotNodesMu sync.Mutex +} + +type NodeDigest struct { + Index int64 // logical index at that layer + Hash [32]byte // 32 bytes +} + +var _ hash.Hash = &Calc{} // make sure we are hash.Hash compliant + +// MaxLayers is the current maximum height of the rust-fil-proofs proving tree. +const MaxLayers = uint(35) // result of log2( 1 TiB / 32 ) + +// MaxPieceSize is the current maximum size of the rust-fil-proofs proving tree. +const MaxPieceSize = uint64(1 << (MaxLayers + 5)) + +// MaxPiecePayload is the maximum amount of data that one can Write() to the +// Calc object, before needing to derive a Digest(). Constrained by the value +// of MaxLayers. +const MaxPiecePayload = MaxPieceSize / 128 * 127 + +// MinPiecePayload is the smallest amount of data for which FR32 padding has +// a defined result. It is not possible to derive a Digest() before Write()ing +// at least this amount of bytes. +const MinPiecePayload = uint64(65) + +const ( + commpDigestSize = sha256simd.Size + quadPayload = 127 + bufferSize = 256 * quadPayload // FIXME: tune better, chosen by rough experiment +) + +var ( + layerQueueDepth = 32 // FIXME: tune better, chosen by rough experiment + shaPool = sync.Pool{New: func() interface{} { return sha256simd.New() }} + stackedNulPadding [MaxLayers][]byte +) + +// initialize the nul padding stack (cheap to do upfront, just MaxLayers loops) +func init() { + h := shaPool.Get().(hash.Hash) + + stackedNulPadding[0] = make([]byte, commpDigestSize) + for i := uint(1); i < MaxLayers; i++ { + h.Reset() + h.Write(stackedNulPadding[i-1]) // yes, got to... + h.Write(stackedNulPadding[i-1]) // ...do it twice + stackedNulPadding[i] = h.Sum(make([]byte, 0, commpDigestSize)) + stackedNulPadding[i][31] &= 0x3F + } + + shaPool.Put(h) +} + +// BlockSize is the amount of bytes consumed by the commP algorithm in one go. +// Write()ing data in multiples of BlockSize would obviate the need to maintain +// an internal carry buffer. The BlockSize of this module is 127 bytes. +func (cp *Calc) BlockSize() int { return quadPayload } + +// Size is the amount of bytes returned on Sum()/Digest(), which is 32 bytes +// for this module. +func (cp *Calc) Size() int { return commpDigestSize } + +// Reset re-initializes the accumulator object, clearing its state and +// terminating all background goroutines. It is safe to Reset() an accumulator +// in any state. +func (cp *Calc) Reset() { + cp.mu.Lock() + if cp.buffer != nil { + // we are resetting without digesting: close everything out to terminate + // the layer workers + close(cp.layerQueues[0]) + <-cp.resultCommP + } + cp.state = state{} // reset + cp.mu.Unlock() +} + +// Sum is a thin wrapper around Digest() and is provided solely to satisfy +// the hash.Hash interface. It panics on errors returned from Digest(). +// Note that unlike classic (hash.Hash).Sum(), calling this method is +// destructive: the internal state is reset and all goroutines kicked off +// by Write() are terminated. +func (cp *Calc) Sum(buf []byte) []byte { + commP, _, err := cp.digest() + if err != nil { + panic(err) + } + return append(buf, commP...) +} + +// Digest collapses the internal hash state and returns the resulting raw 32 +// bytes of commP and the padded piece size, or alternatively an error in +// case of insufficient accumulated state. On success invokes Reset(), which +// terminates all goroutines kicked off by Write(). +func (cp *Calc) digest() (commP []byte, paddedPieceSize uint64, err error) { + cp.mu.Lock() + + defer func() { + // reset only if we did succeed + if err == nil { + cp.state = state{} + } + cp.mu.Unlock() + }() + + if processed := cp.quadsEnqueued*quadPayload + uint64(len(cp.buffer)); processed < MinPiecePayload { + err = xerrors.Errorf( + "insufficient state accumulated: commP is not defined for inputs shorter than %d bytes, but only %d processed so far", + MinPiecePayload, processed, + ) + return + } + + // If any, flush remaining bytes padded up with zeroes + if len(cp.buffer) > 0 { + if mod := len(cp.buffer) % quadPayload; mod != 0 { + cp.buffer = append(cp.buffer, make([]byte, quadPayload-mod)...) + } + for len(cp.buffer) > 0 { + // FIXME: there is a smarter way to do this instead of 127-at-a-time, + // but that's for another PR + cp.digestQuads(cp.buffer[:127]) + cp.buffer = cp.buffer[127:] + } + } + + // This is how we signal to the bottom of the stack that we are done + // which in turn collapses the rest all the way to resultCommP + close(cp.layerQueues[0]) + + paddedPieceSize = cp.quadsEnqueued * 128 + // hacky round-up-to-next-pow2 + if bits.OnesCount64(paddedPieceSize) != 1 { + paddedPieceSize = 1 << uint(64-bits.LeadingZeros64(paddedPieceSize)) + } + + return <-cp.resultCommP, paddedPieceSize, nil +} + +// Write adds bytes to the accumulator, for a subsequent Digest(). Upon the +// first call of this method a few goroutines are started in the background to +// service each layer of the digest tower. If you wrote some data and then +// decide to abandon the object without invoking Digest(), you need to call +// Reset() to terminate all remaining background workers. Unlike a typical +// (hash.Hash).Write, calling this method can return an error when the total +// amount of bytes is about to go over the maximum currently supported by +// Filecoin. +func (cp *Calc) Write(input []byte) (int, error) { + if len(input) == 0 { + return 0, nil + } + + cp.mu.Lock() + defer cp.mu.Unlock() + + if MaxPiecePayload < + (cp.quadsEnqueued*quadPayload)+ + uint64(len(input)) { + return 0, xerrors.Errorf( + "writing additional %d bytes to the accumulator would overflow the maximum supported unpadded piece size %d", + len(input), MaxPiecePayload, + ) + } + + // just starting: initialize internal state, start first background layer-goroutine + if cp.buffer == nil { + cp.buffer = make([]byte, 0, bufferSize) + cp.resultCommP = make(chan []byte, 1) + cp.layerQueues[0] = make(chan []byte, layerQueueDepth) + cp.addLayer(0) + } + + // short Write() - just buffer it + if len(cp.buffer)+len(input) < bufferSize { + cp.buffer = append(cp.buffer, input...) + return len(input), nil + } + + totalInputBytes := len(input) + + if toSplice := bufferSize - len(cp.buffer); toSplice < bufferSize { + cp.buffer = append(cp.buffer, input[:toSplice]...) + input = input[toSplice:] + + cp.digestQuads(cp.buffer) + cp.buffer = cp.buffer[:0] + } + + for len(input) >= bufferSize { + cp.digestQuads(input[:bufferSize]) + input = input[bufferSize:] + } + + if len(input) > 0 { + cp.buffer = append(cp.buffer, input...) + } + + return totalInputBytes, nil +} + +// always called with power-of-2 amount of quads +func (cp *Calc) digestQuads(inSlab []byte) { + + quadsCount := len(inSlab) / 127 + cp.quadsEnqueued += uint64(quadsCount) + outSlab := make([]byte, quadsCount*128) + + for j := 0; j < quadsCount; j++ { + // Cycle over four(4) 31-byte groups, leaving 1 byte in between: + // 31 + 1 + 31 + 1 + 31 + 1 + 31 = 127 + input := inSlab[j*127 : (j+1)*127] + expander := outSlab[j*128 : (j+1)*128] + inputPlus1, expanderPlus1 := input[1:], expander[1:] + + // First 31 bytes + 6 bits are taken as-is (trimmed later) + // Note that copying them into the expansion buffer is mandatory: + // we will be feeding it to the workers which reuse the bottom half + // of the chunk for the result + copy(expander[:], input[:32]) + + // first 2-bit "shim" forced into the otherwise identical bitstream + expander[31] &= 0x3F + + // In: {{ C[7] C[6] }} X[7] X[6] X[5] X[4] X[3] X[2] X[1] X[0] Y[7] Y[6] Y[5] Y[4] Y[3] Y[2] Y[1] Y[0] Z[7] Z[6] Z[5]... + // Out: X[5] X[4] X[3] X[2] X[1] X[0] C[7] C[6] Y[5] Y[4] Y[3] Y[2] Y[1] Y[0] X[7] X[6] Z[5] Z[4] Z[3]... + for i := 31; i < 63; i++ { + expanderPlus1[i] = inputPlus1[i]<<2 | input[i]>>6 + } + + // next 2-bit shim + expander[63] &= 0x3F + + // In: {{ C[7] C[6] C[5] C[4] }} X[7] X[6] X[5] X[4] X[3] X[2] X[1] X[0] Y[7] Y[6] Y[5] Y[4] Y[3] Y[2] Y[1] Y[0] Z[7] Z[6] Z[5]... + // Out: X[3] X[2] X[1] X[0] C[7] C[6] C[5] C[4] Y[3] Y[2] Y[1] Y[0] X[7] X[6] X[5] X[4] Z[3] Z[2] Z[1]... + for i := 63; i < 95; i++ { + expanderPlus1[i] = inputPlus1[i]<<4 | input[i]>>4 + } + + // next 2-bit shim + expander[95] &= 0x3F + + // In: {{ C[7] C[6] C[5] C[4] C[3] C[2] }} X[7] X[6] X[5] X[4] X[3] X[2] X[1] X[0] Y[7] Y[6] Y[5] Y[4] Y[3] Y[2] Y[1] Y[0] Z[7] Z[6] Z[5]... + // Out: X[1] X[0] C[7] C[6] C[5] C[4] C[3] C[2] Y[1] Y[0] X[7] X[6] X[5] X[4] X[3] X[2] Z[1] Z[0] Y[7]... + for i := 95; i < 126; i++ { + expanderPlus1[i] = inputPlus1[i]<<6 | input[i]>>2 + } + + // the final 6 bit remainder is exactly the value of the last expanded byte + expander[127] = input[126] >> 2 + } + + cp.layerQueues[0] <- outSlab +} + +func (cp *Calc) addLayer(myIdx uint) { + // the next layer channel, which we might *not* use + if cp.layerQueues[myIdx+1] != nil { + panic("addLayer called more than once with identical idx argument") + } + cp.layerQueues[myIdx+1] = make(chan []byte, layerQueueDepth) + + go func() { + var twinHold []byte + + for { + slab, queueIsOpen := <-cp.layerQueues[myIdx] + + // the dream is collapsing + if !queueIsOpen { + defer func() { twinHold = nil }() + + // I am last + if myIdx == MaxLayers || cp.layerQueues[myIdx+2] == nil { + cp.resultCommP <- append(make([]byte, 0, 32), twinHold[0:32]...) + return + } + + if twinHold != nil { + copy(twinHold[32:64], stackedNulPadding[myIdx]) + cp.hashSlab254(0, twinHold[0:64]) + cp.layerQueues[myIdx+1] <- twinHold[0:64:64] + } + + // signal the next in line that they are done too + close(cp.layerQueues[myIdx+1]) + return + } + + var pushedWork bool + + switch { + case len(slab) > 1<<(5+myIdx): + cp.hashSlab254(myIdx, slab) + cp.layerQueues[myIdx+1] <- slab + pushedWork = true + case twinHold != nil: + copy(twinHold[32:64], slab[0:32]) + cp.hashSlab254(0, twinHold[0:64]) + cp.layerQueues[myIdx+1] <- twinHold[0:32:64] + pushedWork = true + twinHold = nil + default: + twinHold = slab[0:32:64] + } + + // Check whether we need another worker + // + // n.b. we will not blow out of the preallocated layerQueues array, + // as we disallow Write()s above a certain threshold + if pushedWork && cp.layerQueues[myIdx+2] == nil { + cp.addLayer(myIdx + 1) + } + } + }() +} + +func (cp *Calc) hashSlab254(layerIdx uint, slab []byte) { + h := shaPool.Get().(hash.Hash) + collectSnapshot := int(layerIdx) == cp.snapShotLayerIdx + + stride := 1 << (5 + layerIdx) + for i := 0; len(slab) > i+stride; i += 2 * stride { + h.Reset() + h.Write(slab[i : i+32]) + h.Write(slab[i+stride : 32+i+stride]) + h.Sum(slab[i:i])[31] &= 0x3F // callers expect we will reuse-reduce-recycle + + if collectSnapshot { + d := make([]byte, 32) + copy(d, slab[i:i+32]) + cp.snapshotNodesMu.Lock() + cp.snapshotNodes = append(cp.snapshotNodes, NodeDigest{ + Index: int64(i / 32), // logical index at this layer + Hash: [32]byte(d), + }) + cp.snapshotNodesMu.Unlock() + } + } + + shaPool.Put(h) +} + +func NewCommPWithSize(size uint64) *Calc { + c := new(Calc) + c.state.size = size + + c.snapShotLayerIdx = snapshotLayerIndex(size) + + return c +} + +const ( + targetReadSize = 4 * 1024 * 1024 // 4 MiB + inputBytesPerLeaf = 127 // raw input bytes that become one 32-byte leaf +) + +func snapshotLayerIndex(size uint64) int { + if size == 0 { + panic("size must be > 0") + } + + // Total number of leaves, each representing 127 bytes of input + numLeaves := size / inputBytesPerLeaf + + // What is the top layer index (leaf layer = 0) + leafLayer := bits.Len64(numLeaves - 1) // ceil(log2) + + // At layer `i`, each node spans 2^i leaves + // Each leaf = 127 bytes ā‡’ node at layer i = 127 * 2^i + // Want: 127 * 2^i ā‰ˆ 4 MiB + // So: i = log2(4 MiB / 127) + targetSpanLeaves := targetReadSize / inputBytesPerLeaf + layerDelta := bits.Len64(uint64(targetSpanLeaves - 1)) + + return leafLayer - layerDelta +} + +func (cp *Calc) DigestWithSnapShot() ([]byte, uint64, int, []NodeDigest, error) { + commp, paddedPieceSize, err := cp.digest() + if err != nil { + return nil, 0, 0, nil, err + } + + cp.snapshotNodesMu.Lock() + defer cp.snapshotNodesMu.Unlock() + + out := make([]NodeDigest, len(cp.snapshotNodes)) + copy(out, cp.snapshotNodes) + return commp, paddedPieceSize, cp.snapShotLayerIdx, out, nil +} diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index 9a29f94db..17275d81f 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -8,14 +8,11 @@ import ( "net/url" "time" - "github.com/google/uuid" - "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" @@ -75,42 +72,26 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo idStr := chunks[0].ID - var isMk20 bool - var id ulid.ULID - var uid uuid.UUID - uid, err = uuid.Parse(idStr) + id, err := ulid.Parse(idStr) if err != nil { - serr := err - id, err = ulid.Parse(idStr) - if err != nil { - return false, xerrors.Errorf("parsing deal ID: %w, %w", serr, err) - } - isMk20 = true + return false, xerrors.Errorf("parsing deal ID: %w", err) } - var rawSize int64 - var pcid, pcid2 cid.Cid - var psize abi.PaddedPieceSize - var deal *mk20.Deal + deal, err := mk20.DealFromDB(ctx, a.db, id) + if err != nil { + return false, xerrors.Errorf("getting deal details: %w", err) + } - if isMk20 { - deal, err = mk20.DealFromDB(ctx, a.db, id) - if err != nil { - return false, xerrors.Errorf("getting deal details: %w", err) - } - pi, err := deal.PieceInfo() - if err != nil { - return false, xerrors.Errorf("getting piece info: %w", err) - } - rawSize = int64(pi.RawSize) - pcid = pi.PieceCIDV1 - psize = pi.Size - pcid2 = deal.Data.PieceCID - } else { - rawSize = 4817498192 // TODO: Fix this for PDP - fmt.Println(uid) + pi, err := deal.PieceInfo() + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) } + rawSize := int64(pi.RawSize) + pcid := pi.PieceCIDV1 + psize := pi.Size + pcid2 := deal.Data.PieceCID + var readers []io.Reader var refIds []int64 @@ -218,7 +199,8 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo // Update DB status of piece, deal, PDP comm, err = a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - if isMk20 { + // Update PoRep pipeline + if deal.Products.DDOV1 != nil { spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) if err != nil { return false, fmt.Errorf("getting provider ID: %w", err) @@ -289,10 +271,25 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo if err != nil { return false, xerrors.Errorf("deleting parked piece refs: %w", err) } - } else { - return false, xerrors.Errorf("not implemented for PDP") - // TODO: Do what is required for PDP } + + // Update PDP pipeline + if deal.Products.PDPV1 != nil { + pdp := deal.Products.PDPV1 + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + } + } + return true, nil }, harmonydb.OptionRetry()) diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 23b24c2ad..67f456368 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -232,8 +232,8 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 refIds = append(refIds, refID) } - n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) VALUES ($1, $2, $3, $4)`, - dealID, pi.PieceCIDV1.String(), pi.Size, refIds) + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) VALUES ($1, $2, $3, $4, $5)`, + dealID, pi.PieceCIDV1.String(), pi.Size, mk20.ProductNameDDOV1, refIds) if err != nil { return xerrors.Errorf("inserting mk20 download pipeline: %w", err) } @@ -331,15 +331,15 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 SELECT id, $4, $5, FALSE FROM selected_piece RETURNING ref_id ) - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - VALUES ($6, $1, $2, ARRAY[(SELECT ref_id FROM inserted_ref)]) - ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) + VALUES ($6, $1, $2, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) + ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE SET ref_ids = array_append( market_mk20_download_pipeline.ref_ids, (SELECT ref_id FROM inserted_ref) ) WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, - k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID) + k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, mk20.ProductNameDDOV1) } if batch.Len() > batchSize { @@ -393,8 +393,6 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 return nil } - // Insert pipeline when data - return xerrors.Errorf("unknown data source type") } @@ -483,12 +481,12 @@ func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK2 err = tx.QueryRow(`SELECT u.ref_id FROM ( SELECT unnest(dp.ref_ids) AS ref_id FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 ) u JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id JOIN parked_pieces pp ON pp.id = pr.piece_id WHERE pp.complete = TRUE - LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&refid) + LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&refid) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil @@ -496,8 +494,20 @@ func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK2 return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) } - _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3`, - piece.ID, piece.PieceCID, piece.PieceSize) + // Remove other ref_ids from piece_park_refs + _, err = tx.Exec(`DELETE FROM parked_piece_refs + WHERE ref_id IN ( + SELECT unnest(dp.ref_ids) + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + ) + AND ref_id != $5;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1, refid) + if err != nil { + return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, + piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1) if err != nil { return false, xerrors.Errorf("failed to delete piece from download table: %w", err) } @@ -532,7 +542,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie if piece.Offline && !piece.Downloaded && !piece.Started { comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { var updated bool - err = tx.QueryRow(`SELECT process_offline_download($1, $2, $3)`, piece.ID, piece.PieceCID, piece.PieceSize).Scan(&updated) + err = tx.QueryRow(`SELECT process_offline_download($1, $2, $3, $4)`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&updated) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("failed to start download for offline deal %s: %w", piece.ID, err) @@ -627,10 +637,10 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie RETURNING ref_id ), upsert_pipeline AS ( - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, ref_ids) - SELECT $1, $2, $3, array_agg(ref_id) + INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) + SELECT $1, $2, $3, $7, array_agg(ref_id) FROM inserted_ref - ON CONFLICT (id, piece_cid, piece_size) DO UPDATE + ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE SET ref_ids = ( SELECT array( SELECT DISTINCT r @@ -641,7 +651,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie UPDATE market_mk20_pipeline SET started = TRUE WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND started = FALSE;`, - piece.ID, piece.PieceCID, piece.PieceSize, rawSize, urlString, hdrs) + piece.ID, piece.PieceCID, piece.PieceSize, rawSize, urlString, hdrs, mk20.ProductNameDDOV1) if err != nil { return false, xerrors.Errorf("failed to start download for offline deal using PieceLocator: %w", err) } diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index 86b8aede0..de0aeb8b8 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -25,7 +25,7 @@ import ( "github.com/filecoin-project/curio/market/mk20" ) -type AggregateTask struct { +type AggregateDealTask struct { sm *CurioStorageDealMarket db *harmonydb.DB sc *ffi.SealCalls @@ -33,8 +33,8 @@ type AggregateTask struct { api headAPI } -func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.SealCalls, stor paths.StashStore, api headAPI) *AggregateTask { - return &AggregateTask{ +func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.SealCalls, stor paths.StashStore, api headAPI) *AggregateDealTask { + return &AggregateDealTask{ sm: sm, db: db, sc: sc, @@ -43,7 +43,7 @@ func NewAggregateTask(sm *CurioStorageDealMarket, db *harmonydb.DB, sc *ffi.Seal } } -func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() var pieces []struct { @@ -321,12 +321,12 @@ func (a *AggregateTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d return true, nil } -func (a *AggregateTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (a *AggregateDealTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { // If no local pieceRef was found then just return first TaskID return &ids[0], nil } -func (a *AggregateTask) TypeDetails() harmonytask.TaskTypeDetails { +func (a *AggregateDealTask) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), Name: "AggregateDeals", @@ -338,9 +338,9 @@ func (a *AggregateTask) TypeDetails() harmonytask.TaskTypeDetails { } } -func (a *AggregateTask) Adder(taskFunc harmonytask.AddTaskFunc) { +func (a *AggregateDealTask) Adder(taskFunc harmonytask.AddTaskFunc) { a.sm.adders[pollerAggregate].Set(taskFunc) } -var _ = harmonytask.Reg(&AggregateTask{}) -var _ harmonytask.TaskInterface = &AggregateTask{} +var _ = harmonytask.Reg(&AggregateDealTask{}) +var _ harmonytask.TaskInterface = &AggregateDealTask{} From b790dae8eb2daabe5ed5c3ef275458ede6450b27 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 15 Jul 2025 23:23:01 +0400 Subject: [PATCH 19/55] openAPI, auth, streamline types and methods --- Makefile | 2 +- cmd/sptool/toolbox_deal_client.go | 25 +- cuhttp/server.go | 4 +- go.mod | 17 +- go.sum | 27 +- .../harmonydb/sql/20250505-market_mk20.sql | 14 +- market/http/http.go | 4 +- market/mk20/ddo_v1.go | 23 +- market/mk20/http/docs.go | 1087 +++++++++++++ market/mk20/http/http.go | 363 +++-- market/mk20/http/info.md | 362 ----- market/mk20/http/swagger.json | 1060 +++++++++++++ market/mk20/http/swagger.yaml | 799 ++++++++++ market/mk20/mk20.go | 120 +- market/mk20/mk20_upload.go | 218 ++- market/mk20/mk20_utils.go | 109 +- market/mk20/mk20gen/gen.go | 1340 ++++++++++++----- market/mk20/pdp_v1.go | 9 +- market/mk20/retrieval_v1.go | 2 +- market/mk20/types.go | 105 +- market/mk20/utils.go | 257 +++- tasks/piece/task_aggregate_chunks.go | 167 +- tasks/storage-market/mk20.go | 2 +- 23 files changed, 4916 insertions(+), 1200 deletions(-) create mode 100644 market/mk20/http/docs.go delete mode 100644 market/mk20/http/info.md create mode 100644 market/mk20/http/swagger.json create mode 100644 market/mk20/http/swagger.yaml diff --git a/Makefile b/Makefile index 1b7c1e3b1..0b662a0d4 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ gen: gensimple .PHONY: gen marketgen: - $(GOCC) run ./market/mk20/mk20gen -pkg ./market/mk20 -output ./market/mk20/http/info.md + swag init -dir market/mk20/http -g http.go -o market/mk20/http --parseDependencyLevel 3 --parseDependency .PHONY: marketgen gensimple: api-gen go-generate cfgdoc-gen docsgen marketgen docsgen-cli diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 83cb30764..385e6b60e 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -1917,25 +1917,24 @@ var mk20DealCmd = &cli.Command{ } log.Debugw("generated deal id", "id", id) - msg, err := id.MarshalBinary() - if err != nil { - return xerrors.Errorf("failed to marshal deal id: %w", err) - } + //msg, err := id.MarshalBinary() + //if err != nil { + // return xerrors.Errorf("failed to marshal deal id: %w", err) + //} - sig, err := n.Wallet.WalletSign(ctx, walletAddr, msg, lapi.MsgMeta{Type: lapi.MTDealProposal}) - if err != nil { - return xerrors.Errorf("failed to sign deal proposal: %w", err) - } + //sig, err := n.Wallet.WalletSign(ctx, walletAddr, msg, lapi.MsgMeta{Type: lapi.MTDealProposal}) + //if err != nil { + // return xerrors.Errorf("failed to sign deal proposal: %w", err) + //} - msgb, err := sig.MarshalBinary() - if err != nil { - return xerrors.Errorf("failed to marshal deal proposal signature: %w", err) - } + //msgb, err := sig.MarshalBinary() + //if err != nil { + // return xerrors.Errorf("failed to marshal deal proposal signature: %w", err) + //} deal := mk20.Deal{ Identifier: id, Client: walletAddr, - Signature: msgb, Data: &d, Products: p, } diff --git a/cuhttp/server.go b/cuhttp/server.go index 8d264fd0b..1d3f00e5a 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -50,11 +50,11 @@ func secureHeaders(csp string) func(http.Handler) http.Handler { case "off": // Do nothing case "self": - w.Header().Set("Content-Security-Policy", "default-src 'self'") + w.Header().Set("Content-Security-Policy", "default-src 'self'; img-src 'self' data: blob:") case "inline": fallthrough default: - w.Header().Set("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'") + w.Header().Set("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' data: blob:") } next.ServeHTTP(w, r) diff --git a/go.mod b/go.mod index b28d5c415..beb66115e 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 + github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.14.0 github.com/multiformats/go-multibase v0.2.0 @@ -96,12 +97,13 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb github.com/stretchr/testify v1.10.0 + github.com/swaggo/http-swagger/v2 v2.0.2 + github.com/swaggo/swag v1.16.4 github.com/triplewz/poseidon v0.0.2 github.com/urfave/cli/v2 v2.27.5 github.com/whyrusleeping/cbor-gen v0.3.1 github.com/yugabyte/gocql v1.6.0-yb-1 github.com/yugabyte/pgx/v5 v5.5.3-yb-2 - github.com/yuin/goldmark v1.4.13 go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 @@ -122,10 +124,9 @@ require ( github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa // indirect + github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/akavel/rsrc v0.10.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect @@ -194,10 +195,10 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.19.3 // indirect - github.com/go-openapi/jsonreference v0.19.4 // indirect - github.com/go-openapi/spec v0.19.11 // indirect - github.com/go-openapi/swag v0.19.11 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/spec v0.20.6 // indirect + github.com/go-openapi/swag v0.19.15 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -280,7 +281,6 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -332,6 +332,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.13 // indirect + github.com/swaggo/files/v2 v2.0.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect diff --git a/go.sum b/go.sum index 51a61bd6b..5e37799f9 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivyb github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa h1:1PPxEyGdIGVkX/kqMvLJ95a1dGS1Sz7tpNEgehEYYt0= github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= @@ -69,9 +71,7 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= @@ -214,6 +214,7 @@ github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLR github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= @@ -438,20 +439,24 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.4 h1:3Vw+rh13uq2JFNxgnMTGE1rnoieU9FmyE1gvnyylsYg= github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.11 h1:ogU5q8dtp3MMPn59a9VRrPKVxvJHEs5P7yNMR5sNnis= github.com/go-openapi/spec v0.19.11/go.mod h1:vqK/dIdLGCosfvYsQV3WfC7N3TiZSnGY2RZKoFK7X28= +github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ= +github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc= github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -1099,6 +1104,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nikkolasg/hexjson v0.1.0 h1:Cgi1MSZVQFoJKYeRpBNEcdF3LB+Zo4fYKsDz7h8uJYQ= github.com/nikkolasg/hexjson v0.1.0/go.mod h1:fbGbWFZ0FmJMFbpCMtJpwb0tudVxSSZ+Es2TsCg57cA= github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= @@ -1356,6 +1362,12 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw= +github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM= +github.com/swaggo/http-swagger/v2 v2.0.2 h1:FKCdLsl+sFCx60KFsyM0rDarwiUSZ8DqbfSyIKC9OBg= +github.com/swaggo/http-swagger/v2 v2.0.2/go.mod h1:r7/GBkAWIfK6E/OLnE8fXnviHiDeAHmgIyooa4xm3AQ= +github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= +github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -1444,7 +1456,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -1977,6 +1988,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= @@ -1998,6 +2010,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 8dc8d6d44..8f1376162 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -220,8 +220,11 @@ CREATE TABLE market_mk20_pipeline ( ); CREATE TABLE market_mk20_pipeline_waiting ( - id TEXT PRIMARY KEY, - waiting_for_data BOOLEAN DEFAULT FALSE + id TEXT PRIMARY KEY +); + +CREATE TABLE market_mk20_upload_waiting ( + id TEXT PRIMARY KEY ); CREATE TABLE market_mk20_download_pipeline ( @@ -414,7 +417,7 @@ CREATE TABLE pdp_proofset_root ( remove_message_hash TEXT DEFAULT NULL, remove_message_index BIGINT DEFAULT NULL, - CONSTRAINT pdp_proofset_roots_root_id_unique PRIMARY KEY (proofset, root_id) + PRIMARY KEY (proofset, root) ); CREATE TABLE pdp_pipeline ( @@ -460,4 +463,9 @@ CREATE TABLE pdp_pipeline ( complete BOOLEAN DEFAULT FALSE ); +CREATE TABLE market_mk20_clients ( + client TEXT PRIMARY KEY, + allowed BOOLEAN DEFAULT TRUE +); + diff --git a/market/http/http.go b/market/http/http.go index 79001d0e5..886cca1dd 100644 --- a/market/http/http.go +++ b/market/http/http.go @@ -18,6 +18,7 @@ type MarketHandler struct { mdh12 *mk12http.MK12DealHandler mdh20 *mk20http.MK20DealHandler pdpService *pdp.PDPService + domainName string } // NewMarketHandler is used to prepare all the required market handlers. Currently, it supports mk12 deal market. @@ -44,6 +45,7 @@ func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_mar mdh12: mdh12, mdh20: mdh20, pdpService: pdpService, + domainName: cfg.HTTP.DomainName, }, nil } @@ -51,7 +53,7 @@ func NewMarketHandler(db *harmonydb.DB, cfg *config.CurioConfig, dm *storage_mar // This can include mk12 deals, mk20 deals(WIP), sector market(WIP) etc func Router(mux *chi.Mux, mh *MarketHandler) { mux.Mount("/market/mk12", mk12http.Router(mh.mdh12)) - mux.Mount("/market/mk20", mk20http.Router(mh.mdh20)) + mux.Mount("/market/mk20", mk20http.Router(mh.mdh20, mh.domainName)) if mh.pdpService != nil { mux.Mount("/market/pdp", pdp.Routes(mh.pdpService)) } diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index ed65af516..2db6249b1 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "math/big" - "net/http" "strings" "github.com/ethereum/go-ethereum" @@ -60,7 +59,7 @@ type DDOV1 struct { NotificationPayload []byte `json:"notification_payload"` } -func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { +func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { code, err := IsProductEnabled(db, d.ProductName()) if err != nil { return code, err @@ -74,7 +73,7 @@ func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e for _, m := range cfg.DisabledMiners { maddr, err := address.NewFromString(m) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("failed to parse miner string: %s", err) + return ErrServerInternalError, xerrors.Errorf("failed to parse miner string: %s", err) } mk20disabledMiners = append(mk20disabledMiners, maddr) } @@ -118,11 +117,11 @@ func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e return Ok, nil } -func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, ErrorCode, error) { +func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, DealCode, error) { if d.ContractAddress == "0xtest" { v, err := rand.Int(rand.Reader, big.NewInt(10000000)) if err != nil { - return "", http.StatusInternalServerError, xerrors.Errorf("failed to generate random number: %w", err) + return "", ErrServerInternalError, xerrors.Errorf("failed to generate random number: %w", err) } return v.String(), Ok, nil } @@ -133,12 +132,12 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. if errors.Is(err, pgx.ErrNoRows) { return "", ErrMarketNotEnabled, UnknowContract } - return "", http.StatusInternalServerError, xerrors.Errorf("getting abi: %w", err) + return "", ErrServerInternalError, xerrors.Errorf("getting abi: %w", err) } parsedABI, err := eabi.JSON(strings.NewReader(abiStr)) if err != nil { - return "", http.StatusInternalServerError, xerrors.Errorf("parsing abi: %w", err) + return "", ErrServerInternalError, xerrors.Errorf("parsing abi: %w", err) } to := common.HexToAddress(d.ContractAddress) @@ -146,18 +145,18 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Get the method method, exists := parsedABI.Methods[d.ContractVerifyMethod] if !exists { - return "", http.StatusInternalServerError, fmt.Errorf("method %s not found in ABI", d.ContractVerifyMethod) + return "", ErrServerInternalError, fmt.Errorf("method %s not found in ABI", d.ContractVerifyMethod) } // Enforce method must take exactly one `bytes` parameter if len(method.Inputs) != 1 || method.Inputs[0].Type.String() != "bytes" { - return "", http.StatusInternalServerError, fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) + return "", ErrServerInternalError, fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) } // ABI-encode method call with input callData, err := parsedABI.Pack(method.Name, d.ContractVerifyMethod) if err != nil { - return "", http.StatusInternalServerError, fmt.Errorf("failed to encode call data: %w", err) + return "", ErrServerInternalError, fmt.Errorf("failed to encode call data: %w", err) } // Build call message @@ -169,13 +168,13 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Call contract output, err := eth.CallContract(ctx, msg, nil) if err != nil { - return "", http.StatusInternalServerError, fmt.Errorf("eth_call failed: %w", err) + return "", ErrServerInternalError, fmt.Errorf("eth_call failed: %w", err) } // Decode return value (assume string) var result string if err := parsedABI.UnpackIntoInterface(&result, method.Name, output); err != nil { - return "", http.StatusInternalServerError, fmt.Errorf("decode result: %w", err) + return "", ErrServerInternalError, fmt.Errorf("decode result: %w", err) } if result == "" { diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go new file mode 100644 index 000000000..54add3928 --- /dev/null +++ b/market/mk20/http/docs.go @@ -0,0 +1,1087 @@ +// Package http Code generated by swaggo/swag. DO NOT EDIT +package http + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": {}, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/contracts": { + "get": { + "description": "List of supported DDO contracts", + "summary": "List of supported DDO contracts", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/products": { + "get": { + "description": "List of supported products", + "summary": "List of supported products", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/sources": { + "get": { + "description": "List of supported data sources", + "summary": "List of supported dats sources", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/status/{id}": { + "get": { + "description": "List of supported DDO contracts", + "summary": "List of supported DDO contracts", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/store": { + "post": { + "description": "Make a mk20 deal", + "consumes": [ + "application/json" + ], + "summary": "Make a mk20 deal", + "parameters": [ + { + "description": "mk20.Deal in json format", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/update/{id}": { + "get": { + "description": "Useful for adding adding additional products and updating PoRep duration", + "consumes": [ + "application/json" + ], + "summary": "Update the deal details of existing deals.", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.Deal in json format", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/upload/finalize/{id}": { + "post": { + "description": "Finalizes the upload process once all the chunks are uploaded.", + "consumes": [ + "application/json" + ], + "summary": "Finalizes the upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.deal in json format", + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/upload/{id}": { + "get": { + "description": "Return a json struct detailing the current status of a deal upload.", + "summary": "Status of deal upload", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "UploadStatusCodeOk represents a successful upload operation with status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStatusCodeDealNotFound indicates that the requested deal was not found, corresponding to status code 404", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "425": { + "description": "UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "500": { + "description": "UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + } + } + }, + "post": { + "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", + "summary": "Starts the upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "UploadStartCodeOk indicates a successful upload start request with status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "409": { + "description": "UploadStartCodeAlreadyStarted indicates that the upload process has already been initiated and cannot be started again", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "500": { + "description": "UploadStartCodeServerError indicates an error occurred on the server while processing an upload start request", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + } + } + } + }, + "/upload/{id}/{chunkNum}": { + "put": { + "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", + "summary": "Upload a file chunk", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "chunkNum", + "name": "chunkNum", + "in": "path", + "required": true + }, + { + "description": "raw binary", + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } + } + ], + "responses": { + "200": { + "description": "UploadOk indicates a successful upload operation, represented by the HTTP status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadNotFound represents an error where the requested upload chunk could not be found, typically corresponding to HTTP status 404", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "409": { + "description": "UploadChunkAlreadyUploaded indicates that the chunk has already been uploaded and cannot be re-uploaded", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "500": { + "description": "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + } + } + } + } + }, + "definitions": { + "address.Address": { + "type": "object" + }, + "cid.Cid": { + "type": "object" + }, + "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { + "type": "integer", + "enum": [ + 0 + ], + "x-enum-varnames": [ + "NoAllocationID" + ] + }, + "http.Header": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "mk20.AggregateType": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "x-enum-varnames": [ + "AggregateTypeNone", + "AggregateTypeV1" + ] + }, + "mk20.DDOV1": { + "type": "object", + "properties": { + "allocation_id": { + "description": "AllocationId represents an aggregated allocation identifier for the deal.", + "allOf": [ + { + "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" + } + ] + }, + "contract_address": { + "description": "ContractAddress specifies the address of the contract governing the deal", + "type": "string" + }, + "contract_verify_method": { + "description": "ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract", + "type": "string" + }, + "contract_verify_method_params": { + "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", + "type": "array", + "items": { + "type": "integer" + } + }, + "duration": { + "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", + "type": "integer" + }, + "notification_address": { + "description": "NotificationAddress specifies the address to which notifications will be relayed to when sector is activated", + "type": "string" + }, + "notification_payload": { + "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", + "type": "array", + "items": { + "type": "integer" + } + }, + "piece_manager": { + "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + }, + "provider": { + "description": "Provider specifies the address of the provider", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + } + } + }, + "mk20.DataSource": { + "type": "object", + "properties": { + "format": { + "description": "Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats.", + "allOf": [ + { + "$ref": "#/definitions/mk20.PieceDataFormat" + } + ] + }, + "piece_cid": { + "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", + "allOf": [ + { + "$ref": "#/definitions/cid.Cid" + } + ] + }, + "source_aggregate": { + "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceAggregate" + } + ] + }, + "source_http": { + "description": "SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceHTTP" + } + ] + }, + "source_httpput": { + "description": "SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceHttpPut" + } + ] + }, + "source_offline": { + "description": "SourceOffline defines the data source for offline pieces, including raw size information.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceOffline" + } + ] + } + } + }, + "mk20.DataSourceAggregate": { + "type": "object", + "properties": { + "pieces": { + "type": "array", + "items": { + "$ref": "#/definitions/mk20.DataSource" + } + } + } + }, + "mk20.DataSourceHTTP": { + "type": "object", + "properties": { + "urls": { + "description": "URLs lists the HTTP endpoints where the piece data can be fetched.", + "type": "array", + "items": { + "$ref": "#/definitions/mk20.HttpUrl" + } + } + } + }, + "mk20.DataSourceHttpPut": { + "type": "object" + }, + "mk20.DataSourceOffline": { + "type": "object" + }, + "mk20.Deal": { + "type": "object", + "properties": { + "client": { + "description": "Client wallet for the deal", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + }, + "data": { + "description": "Data represents the source of piece data and associated metadata.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSource" + } + ] + }, + "identifier": { + "description": "Identifier represents a unique identifier for the deal in UUID format.", + "type": "array", + "items": { + "type": "integer" + } + }, + "products": { + "description": "Products represents a collection of product-specific information associated with a deal", + "allOf": [ + { + "$ref": "#/definitions/mk20.Products" + } + ] + } + } + }, + "mk20.DealCode": { + "type": "integer", + "enum": [ + 200, + 401, + 400, + 404, + 430, + 422, + 423, + 424, + 425, + 426, + 500, + 503, + 429, + 440, + 441 + ], + "x-enum-varnames": [ + "Ok", + "ErrUnAuthorized", + "ErrBadProposal", + "ErrDealNotFound", + "ErrMalformedDataSource", + "ErrUnsupportedDataSource", + "ErrUnsupportedProduct", + "ErrProductNotEnabled", + "ErrProductValidationFailed", + "ErrDealRejectedByMarket", + "ErrServerInternalError", + "ErrServiceMaintenance", + "ErrServiceOverloaded", + "ErrMarketNotEnabled", + "ErrDurationTooShort" + ] + }, + "mk20.FormatAggregate": { + "type": "object", + "properties": { + "sub": { + "description": "Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format.\nThe order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate", + "type": "array", + "items": { + "$ref": "#/definitions/mk20.DataSource" + } + }, + "type": { + "description": "Type specifies the type of aggregation for data pieces, represented by an AggregateType value.", + "allOf": [ + { + "$ref": "#/definitions/mk20.AggregateType" + } + ] + } + } + }, + "mk20.FormatBytes": { + "type": "object" + }, + "mk20.FormatCar": { + "type": "object" + }, + "mk20.HttpUrl": { + "type": "object", + "properties": { + "fallback": { + "description": "Fallback indicates whether this URL serves as a fallback option when other URLs fail.", + "type": "boolean" + }, + "headers": { + "description": "HTTPHeaders represents the HTTP headers associated with the URL.", + "allOf": [ + { + "$ref": "#/definitions/http.Header" + } + ] + }, + "priority": { + "description": "Priority indicates the order preference for using the URL in requests, with lower values having higher priority.", + "type": "integer" + }, + "url": { + "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", + "type": "string" + } + } + }, + "mk20.PDPV1": { + "type": "object", + "properties": { + "add_root": { + "description": "AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined.", + "type": "boolean" + }, + "create_proof_set": { + "description": "CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider.", + "type": "boolean" + }, + "delete_proof_set": { + "description": "DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client.\nProofSetID must be defined.", + "type": "boolean" + }, + "delete_root": { + "description": "DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined.", + "type": "boolean" + }, + "extra_data": { + "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", + "type": "array", + "items": { + "type": "integer" + } + }, + "proof_set_id": { + "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", + "type": "integer" + } + } + }, + "mk20.PieceDataFormat": { + "type": "object", + "properties": { + "aggregate": { + "description": "Aggregate holds a reference to the aggregated format of piece data.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatAggregate" + } + ] + }, + "car": { + "description": "Car represents the optional CAR file format, including its metadata and versioning details.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatCar" + } + ] + }, + "raw": { + "description": "Raw represents the raw format of the piece data, encapsulated as bytes.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatBytes" + } + ] + } + } + }, + "mk20.Products": { + "type": "object", + "properties": { + "ddo_v1": { + "description": "DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)", + "allOf": [ + { + "$ref": "#/definitions/mk20.DDOV1" + } + ] + }, + "pdp_v1": { + "description": "PDPV1 represents product-specific configuration for PDP version 1 deals.", + "allOf": [ + { + "$ref": "#/definitions/mk20.PDPV1" + } + ] + }, + "retrieval_v1": { + "description": "RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags.", + "allOf": [ + { + "$ref": "#/definitions/mk20.RetrievalV1" + } + ] + } + } + }, + "mk20.RetrievalV1": { + "type": "object", + "properties": { + "announce_payload": { + "description": "AnnouncePayload indicates whether the payload should be announced to IPNI.", + "type": "boolean" + }, + "announce_piece": { + "description": "AnnouncePiece indicates whether the piece information should be announced to IPNI.", + "type": "boolean" + }, + "indexing": { + "description": "Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval", + "type": "boolean" + } + } + }, + "mk20.UploadCode": { + "type": "integer", + "enum": [ + 200, + 400, + 404, + 409, + 500 + ], + "x-enum-varnames": [ + "UploadOk", + "UploadBadRequest", + "UploadNotFound", + "UploadChunkAlreadyUploaded", + "UploadServerError" + ] + }, + "mk20.UploadStartCode": { + "type": "integer", + "enum": [ + 200, + 400, + 404, + 409, + 500 + ], + "x-enum-varnames": [ + "UploadStartCodeOk", + "UploadStartCodeBadRequest", + "UploadStartCodeDealNotFound", + "UploadStartCodeAlreadyStarted", + "UploadStartCodeServerError" + ] + }, + "mk20.UploadStatusCode": { + "type": "integer", + "enum": [ + 200, + 404, + 425, + 500 + ], + "x-enum-varnames": [ + "UploadStatusCodeOk", + "UploadStatusCodeDealNotFound", + "UploadStatusCodeUploadNotStarted", + "UploadStatusCodeServerError" + ] + } + }, + "securityDefinitions": { + "CurioAuth": { + "description": "Use the format: ` + "`" + `CurioAuth PublicKeyType:PublicKey:Signature` + "`" + `\n\n- ` + "`" + `PublicKeyType` + "`" + `: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- ` + "`" + `PublicKey` + "`" + `: Base64 string of public key bytes\n- ` + "`" + `Signature` + "`" + `: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order.\n- The raw public key bytes (not a human-readable address)\n- The HTTP request path, such as /user/info\n- The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z)\n- These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "", + Host: "", + BasePath: "", + Schemes: []string{}, + Title: "Curio Market 2.0 API", + Description: "Curio market APIs", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 8036e47d8..4ea0413a7 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -1,8 +1,8 @@ package http import ( - "bytes" "context" + "embed" _ "embed" "encoding/json" "errors" @@ -10,18 +10,14 @@ import ( "io" "net/http" "strconv" - "strings" "time" "github.com/go-chi/chi/v5" "github.com/go-chi/httprate" logging "github.com/ipfs/go-log/v2" "github.com/oklog/ulid" + httpSwagger "github.com/swaggo/http-swagger/v2" "github.com/yugabyte/pgx/v5" - "github.com/yuin/goldmark" - "github.com/yuin/goldmark/extension" - "github.com/yuin/goldmark/parser" - "github.com/yuin/goldmark/renderer/html" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -32,11 +28,13 @@ import ( storage_market "github.com/filecoin-project/curio/tasks/storage-market" ) -//go:embed info.md -var infoMarkdown []byte +//go:embed swagger.yaml swagger.json docs.go +var swaggerAssets embed.FS var log = logging.Logger("mk20httphdlr") +const version = "1.0.0" + const requestTimeout = 10 * time.Second type MK20DealHandler struct { @@ -62,9 +60,77 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { return httprate.LimitByIP(50, 1*time.Second) } -func Router(mdh *MK20DealHandler) http.Handler { +func AuthMiddleware(db *harmonydb.DB) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "Missing Authorization header", http.StatusUnauthorized) + return + } + + allowed, client, err := mk20.Auth(authHeader, r.URL.Path, db) + if err != nil { + log.Errorw("failed to authenticate request", "err", err) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + + if !allowed { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + idStr := chi.URLParam(r, "id") + if idStr != "" { + allowed, err := mk20.AuthenticateClient(db, idStr, client) + if err != nil { + log.Errorw("failed to authenticate client", "err", err) + http.Error(w, err.Error(), http.StatusUnauthorized) + return + } + if !allowed { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + } + + next.ServeHTTP(w, r) + }) + } +} + +// @title Curio Market 2.0 API +// @description Curio market APIs +func Router(mdh *MK20DealHandler, domainName string) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) + mux.Mount("/", APIRouter(mdh, domainName)) + mux.Mount("/info", InfoRouter()) + return mux +} + +// @securityDefinitions.apikey CurioAuth +// @in header +// @name Authorization +// @description Use the format: `CurioAuth PublicKeyType:PublicKey:Signature` +// @description +// @description - `PublicKeyType`: String representation of type of wallet (e.g., "ed25519", "bls", "secp256k1") +// @description - `PublicKey`: Base64 string of public key bytes +// @description - `Signature`: Signature is Base64 string of signature bytes. +// @description - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order. +// @description - The raw public key bytes (not a human-readable address) +// @description - The HTTP request path, such as /user/info +// @description - The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z) +// @description - These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. +// @security CurioAuth +func APIRouter(mdh *MK20DealHandler, domainName string) http.Handler { + SwaggerInfo.BasePath = "/market/mk20" + SwaggerInfo.Host = fmt.Sprintf("https://%s", domainName) + SwaggerInfo.Version = version + mux := chi.NewRouter() + mux.Use(dealRateLimitMiddleware()) + mux.Use(AuthMiddleware(mdh.db)) mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "request timeout")) mux.Method("GET", "/status/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "request timeout")) mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) @@ -72,16 +138,63 @@ func Router(mdh *MK20DealHandler) http.Handler { mux.Method("GET", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStatus), requestTimeout, "request timeout")) mux.Put("/upload/{id}/{chunkNum}", mdh.mk20UploadDealChunks) mux.Method("POST", "/upload/finalize/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20FinalizeUpload), requestTimeout, "request timeout")) - mux.Method("GET", "/info", http.TimeoutHandler(http.HandlerFunc(mdh.info), requestTimeout, "request timeout")) mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) - mux.Method("GET", "/update", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UpdateDeal), requestTimeout, "request timeout")) + mux.Method("POST", "/update/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UpdateDeal), requestTimeout, "request timeout")) + return mux +} + +// InfoRouter serves OpenAPI specs and OpenAPI info +func InfoRouter() http.Handler { + mux := chi.NewRouter() + mux.Get("/*", httpSwagger.Handler()) + + mux.Get("/swagger.yaml", func(w http.ResponseWriter, r *http.Request) { + swaggerYAML, err := swaggerAssets.ReadFile("swagger.yaml") + if err != nil { + log.Errorw("failed to read swagger.yaml", "err", err) + http.Error(w, "failed to read swagger.yaml", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/x-yaml") + _, _ = w.Write(swaggerYAML) + }) + + mux.Get("/swagger.json", func(w http.ResponseWriter, r *http.Request) { + swaggerJSON, err := swaggerAssets.ReadFile("swagger.json") + if err != nil { + log.Errorw("failed to read swagger.json", "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(swaggerJSON) + }) return mux } -// mk20deal handles incoming HTTP POST requests to process MK20 deals. -// It validates the request's content type and body, then parses and executes the deal logic. -// Responds with appropriate HTTP status codes and logs detailed information about the process. +// mk20deal handles HTTP requests to process MK20 deals, parses the request body, validates it, and executes the deal logic. +// @Router /store [post] +// @Summary Make a mk20 deal +// @Description Make a mk20 deal +// @BasePath /market/mk20 +// @Param body body mk20.Deal true "mk20.Deal in json format" +// @Accept json +// @Failure 200 {object} mk20.DealCode "Ok represents a successful operation with an HTTP status code of 200" +// @Failure 400 {object} mk20.DealCode "ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic" +// @Failure 404 {object} mk20.DealCode "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404" +// @Failure 430 {object} mk20.DealCode "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data" +// @Failure 422 {object} mk20.DealCode "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context" +// @Failure 423 {object} mk20.DealCode "ErrUnsupportedProduct indicates that the requested product is not supported by the provider" +// @Failure 424 {object} mk20.DealCode "ErrProductNotEnabled indicates that the requested product is not enabled on the provider" +// @Failure 425 {object} mk20.DealCode "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data" +// @Failure 426 {object} mk20.DealCode "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules" +// @Failure 500 {object} mk20.DealCode "ErrServerInternalError indicates an internal server error with a corresponding error code of 500" +// @Failure 503 {object} mk20.DealCode "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503" +// @Failure 429 {object} mk20.DealCode "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment" +// @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" +// @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") var deal mk20.Deal @@ -117,14 +230,22 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { "HTTPCode", result.HTTPCode, "Reason", result.Reason) - w.WriteHeader(result.HTTPCode) + w.WriteHeader(int(result.HTTPCode)) _, err = w.Write([]byte(fmt.Sprint("Reason: ", result.Reason))) if err != nil { log.Errorw("writing deal response:", "id", deal.Identifier, "error", err) } } -// mk20status handles HTTP requests to retrieve the status of a deal using its ID, responding with deal status or appropriate error codes. +// mk20status handles HTTP requests to fetch the status of a deal by its ID and responding with JSON-encoded results. +// @Router /status/{id} [get] +// @Summary List of supported DDO contracts +// @Description List of supported DDO contracts +// @BasePath /market/mk20 +// @Param id path string true "id" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @Failure 200 {string} string "OK - Success" +// @Failure 500 {string} string "Internal Server Error" func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -160,7 +281,13 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { } } -// mk20supportedContracts retrieves supported contract addresses from the database and returns them as a JSON response. +// mk20supportedContracts handles HTTP requests to retrieve supported contract addresses and returns them in a JSON response. +// @Router /contracts [get] +// @Summary List of supported DDO contracts +// @Description List of supported DDO contracts +// @BasePath /market/mk20 +// @Failure 500 {string} string "Internal Server Error" +// @Failure 200 {string} string "OK - Success" func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *http.Request) { var contracts mk20.SupportedContracts err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM ddo_contracts") @@ -189,106 +316,13 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt } } -// info serves the contents of the info file as a text/markdown response with HTTP 200 or returns an HTTP 500 on read/write failure. -func (mdh *MK20DealHandler) info(w http.ResponseWriter, r *http.Request) { - - prods, srcs, err := mdh.dm.MK20Handler.Supported(r.Context()) - if err != nil { - log.Errorw("failed to get supported producers and sources", "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - - var sb strings.Builder - - sb.WriteString(`

    Supported Products

    -
    Piece IndexPiece CID V2 Piece CID Piece Size Deal ID
    ${piece.PieceIndex}${piece.PieceCid} + ${piece.PieceCidV2 && piece.PieceCidV2.trim() !== "" + ? html`${piece.PieceCidV2}` + : 'NA'} + ${piece.PieceCid} ${piece.PieceSize} ${piece.DealID} ${piece.DataUrl}
    -`) - - for name, enabled := range prods { - status := "Disabled" - if enabled { - status = "Enabled" - } - sb.WriteString(fmt.Sprintf("", name, status)) - } - sb.WriteString(`
    NameStatus
    %s%s
    `) - - sb.WriteString(`

    Supported Data Sources

    - -`) - - for name, enabled := range srcs { - status := "Disabled" - if enabled { - status = "Enabled" - } - sb.WriteString(fmt.Sprintf("", name, status)) - } - sb.WriteString(`
    NameStatus
    %s%s
    `) - - summaryHTML := sb.String() - - var mdRenderer = goldmark.New( - goldmark.WithExtensions( - extension.GFM, - extension.Linkify, - extension.Table, - extension.DefinitionList, - ), - goldmark.WithRendererOptions( - html.WithHardWraps(), - html.WithXHTML(), - ), - goldmark.WithParserOptions( - parser.WithAutoHeadingID(), - ), - ) - - var buf bytes.Buffer - if err := mdRenderer.Convert(infoMarkdown, &buf); err != nil { - http.Error(w, "failed to render markdown", http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - - renderedMarkdown := strings.ReplaceAll(buf.String(), "", `
    `) - rendered := summaryHTML + renderedMarkdown - - htmlStr := fmt.Sprintf(` - - - - - Curio Deal Schema - - - - - -
    - %s -
    - - `, rendered) - - _, err = w.Write([]byte(htmlStr)) - if err != nil { - log.Errorw("failed to write info file", "err", err) - } -} - +// supportedProducts handles HTTP requests to retrieve a list of supported MK20 products and returns them in a JSON response. +// @Router /products [get] +// @Summary List of supported products +// @Description List of supported products +// @BasePath /market/mk20 +// @Failure 500 {string} string "Internal Server Error" +// @Failure 200 {string} string "OK - Success" func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Request) { prods, _, err := mdh.dm.MK20Handler.Supported(r.Context()) if err != nil { @@ -316,6 +350,13 @@ func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Req } } +// supportedDataSources handles HTTP requests to retrieve the supported data sources in JSON format. +// @Router /sources [get] +// @Summary List of supported dats sources +// @Description List of supported data sources +// @BasePath /market/mk20 +// @Failure 500 {string} string "Internal Server Error" +// @Failure 200 {string} string "OK - Success" func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http.Request) { _, srcs, err := mdh.dm.MK20Handler.Supported(r.Context()) if err != nil { @@ -343,6 +384,17 @@ func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http. } } +// mk20UploadStatus handles the upload status requests for a given id. +// @Router /upload/{id} [get] +// @Param id path string true "id" +// @Summary Status of deal upload +// @Description Return a json struct detailing the current status of a deal upload. +// @BasePath /market/mk20 +// @Failure 200 {object} mk20.UploadStatusCode "UploadStatusCodeOk represents a successful upload operation with status code 200" +// @Failure 404 {object} mk20.UploadStatusCode "UploadStatusCodeDealNotFound indicates that the requested deal was not found, corresponding to status code 404" +// @Failure 425 {object} mk20.UploadStatusCode "UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet" +// @Failure 500 {object} mk20.UploadStatusCode "UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -359,6 +411,21 @@ func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Requ mdh.dm.MK20Handler.HandleUploadStatus(r.Context(), id, w) } +// mk20UploadDealChunks handles uploading of deal file chunks. +// @Router /upload/{id}/{chunkNum} [put] +// @Summary Upload a file chunk +// @Description Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads. +// @BasePath /market/mk20 +// @Param id path string true "id" +// @Param chunkNum path string true "chunkNum" +// @accepts bytes +// @Param body body []byte true "raw binary" +// @Failure 200 {object} mk20.UploadCode "UploadOk indicates a successful upload operation, represented by the HTTP status code 200" +// @Failure 400 {object} mk20.UploadCode "UploadBadRequest represents a bad request error with an HTTP status code of 400" +// @Failure 404 {object} mk20.UploadCode "UploadNotFound represents an error where the requested upload chunk could not be found, typically corresponding to HTTP status 404" +// @Failure 409 {object} mk20.UploadCode "UploadChunkAlreadyUploaded indicates that the chunk has already been uploaded and cannot be re-uploaded" +// @Failure 500 {object} mk20.UploadCode "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") if ct != "application/octet-stream" { @@ -397,6 +464,18 @@ func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http. mdh.dm.MK20Handler.HandleUploadChunk(id, chunkNum, r.Body, w) } +// mk20UploadStart handles the initiation of an upload process for MK20 deal data. +// @Router /upload/{id} [post] +// @Summary Starts the upload process +// @Description Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal. +// @BasePath /market/mk20 +// @Param id path string true "id" +// @Failure 200 {object} mk20.UploadStartCode "UploadStartCodeOk indicates a successful upload start request with status code 200" +// @Failure 400 {object} mk20.UploadStartCode "UploadStartCodeBadRequest indicates a bad upload start request error with status code 400" +// @Failure 404 {object} mk20.UploadStartCode "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process" +// @Failure 409 {object} mk20.UploadStartCode "UploadStartCodeAlreadyStarted indicates that the upload process has already been initiated and cannot be started again" +// @Failure 500 {object} mk20.UploadStartCode "UploadStartCodeServerError indicates an error occurred on the server while processing an upload start request" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") if ct != "application/json" { @@ -435,10 +514,34 @@ func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Reque return } - mdh.dm.MK20Handler.HandleUploadStart(r.Context(), id, upload.ChunkSize, w) + mdh.dm.MK20Handler.HandleUploadStart(r.Context(), id, upload, w) } +// mk20FinalizeUpload finalizes the upload process for a given deal by processing the request and updating the associated deal in the system if required. +// @Router /upload/finalize/{id} [post] +// @Summary Finalizes the upload process +// @Description Finalizes the upload process once all the chunks are uploaded. +// @BasePath /market/mk20 +// @Param id path string true "id" +// @accepts json +// @Param body body mk20.Deal optional "mk20.deal in json format" +// @Accept json +// @Failure 200 {object} mk20.DealCode "Ok represents a successful operation with an HTTP status code of 200" +// @Failure 400 {object} mk20.DealCode "ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic" +// @Failure 404 {object} mk20.DealCode "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404" +// @Failure 430 {object} mk20.DealCode "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data" +// @Failure 422 {object} mk20.DealCode "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context" +// @Failure 423 {object} mk20.DealCode "ErrUnsupportedProduct indicates that the requested product is not supported by the provider" +// @Failure 424 {object} mk20.DealCode "ErrProductNotEnabled indicates that the requested product is not enabled on the provider" +// @Failure 425 {object} mk20.DealCode "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data" +// @Failure 426 {object} mk20.DealCode "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules" +// @Failure 500 {object} mk20.DealCode "ErrServerInternalError indicates an internal server error with a corresponding error code of 500" +// @Failure 503 {object} mk20.DealCode "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503" +// @Failure 429 {object} mk20.DealCode "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment" +// @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" +// @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -489,6 +592,30 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re mdh.dm.MK20Handler.HandleUploadFinalize(id, &deal, w) } +// mk20UpdateDeal handles updating an MK20 deal based on the provided HTTP request. +// It validates the deal ID, request content type, and JSON body before updating. +// @Summary Update the deal details of existing deals. +// @Description Useful for adding adding additional products and updating PoRep duration +// @BasePath /market/mk20 +// @Router /update/{id} [get] +// @Param id path string true "id" +// @Accept json +// @Param body body mk20.Deal true "mk20.Deal in json format" +// @Failure 200 {object} mk20.DealCode "Ok represents a successful operation with an HTTP status code of 200" +// @Failure 400 {object} mk20.DealCode "ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic" +// @Failure 404 {object} mk20.DealCode "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404" +// @Failure 430 {object} mk20.DealCode "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data" +// @Failure 422 {object} mk20.DealCode "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context" +// @Failure 423 {object} mk20.DealCode "ErrUnsupportedProduct indicates that the requested product is not supported by the provider" +// @Failure 424 {object} mk20.DealCode "ErrProductNotEnabled indicates that the requested product is not enabled on the provider" +// @Failure 425 {object} mk20.DealCode "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data" +// @Failure 426 {object} mk20.DealCode "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules" +// @Failure 500 {object} mk20.DealCode "ErrServerInternalError indicates an internal server error with a corresponding error code of 500" +// @Failure 503 {object} mk20.DealCode "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503" +// @Failure 429 {object} mk20.DealCode "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment" +// @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" +// @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { diff --git a/market/mk20/http/info.md b/market/mk20/http/info.md deleted file mode 100644 index 01b13b295..000000000 --- a/market/mk20/http/info.md +++ /dev/null @@ -1,362 +0,0 @@ -# Storage Market Interface - -This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider. - -## šŸ“” MK20 HTTP API Overview - -The MK20 storage market module provides a set of HTTP endpoints under `/market/mk20` that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior. - -### Base URL - -The base URL for all MK20 endpoints is: - -``` - -/market/mk20 - -``` - -### šŸ”„ POST /store - -Submit a new MK20 deal. - -- **Content-Type**: N/A -- **Body**: N/A -- **Query Parameters**: N/A -- **Response**: - - `200 OK`: Deal accepted - - Other [HTTP codes](#constants-for-errorcode) indicate validation failure, rejection, or system errors - -### šŸ§¾ GET /status?id= - -Retrieve the current status of a deal. - -- **Content-Type**: `application/json` -- **Body**: N/A -- **Query Parameters**: - - `id`: Deal identifier in [ULID](https://github.com/ulid/spec) format -- **Response**: - - `200 OK`: JSON-encoded [deal status](#dealstatusresponse) information - - `400 Bad Request`: Missing or invalid ID - - `500 Internal Server Error`: If backend fails to respond - -### šŸ“œ GET /contracts - -- **Content-Type**: N/A -- **Body**: N/A -- **Query Parameters**: N/A -Return the list of contract addresses supported by the provider. - -- **Response**: - - `200 OK`: [JSON array of contract addresses](#supportedcontracts) - - `500 Internal Server Error`: Query or serialization failure - -### šŸ—‚ PUT /data?id= - -Upload deal data after the deal has been accepted. - -- **Content-Type**: `application/octet-stream` -- **Body**: Deal data bytes -- **Query Parameter**: - -`id`: Deal identifier in [ULID](https://github.com/ulid/spec) format -- **Headers**: - - `Content-Length`: must be deal's raw size -- **Response**: - - `200 OK`: if data is successfully streamed - - `400`, `413`, or `415`: on validation failures - -### šŸ§  GET /info - -- **Content-Type**: N/A -- **Body**: N/A -- **Query Parameters**: N/A -Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources. - -- **Response**: - - `200 OK`: with markdown content of the info file - - `500 Internal Server Error`: if file is not found or cannot be read - -### šŸ§° GET /products - -- **Content-Type**: N/A -- **Body**: N/A -- **Query Parameters**: N/A -Fetch json list of the supported products. - -- **Response**: - - `200 OK`: with json content - - `500 Internal Server Error`: if info cannot be read - -### šŸŒ GET /sources - -- **Content-Type**: N/A -- **Body**: N/A -- **Query Parameters**: N/A -Fetch json list of the supported data sources. - -- **Response**: - - `200 OK`: with json content - - `500 Internal Server Error`: if info cannot be read - -## Supported Deal Types - -This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission. - -### Deal - -Deal represents a structure defining the details and components of a specific deal in the system. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Identifier | [ulid.ULID](https://pkg.go.dev/github.com/oklog/ulid#ULID) | json:"identifier" | Identifier represents a unique identifier for the deal in UUID format. | -| Client | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"client" | Client wallet for the deal | -| Signature | [[]byte](https://pkg.go.dev/builtin#byte) | json:"signature" | Signature bytes for the client deal | -| Data | [*mk20.DataSource](#datasource) | json:"data" | Data represents the source of piece data and associated metadata. | -| Products | [mk20.Products](#products) | json:"products" | Products represents a collection of product-specific information associated with a deal | - -### DataSource - -DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| PieceCID | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. | -| Format | [mk20.PieceDataFormat](#piecedataformat) | json:"format" | Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. | -| SourceHTTP | [*mk20.DataSourceHTTP](#datasourcehttp) | json:"source_http" | SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. | -| SourceAggregate | [*mk20.DataSourceAggregate](#datasourceaggregate) | json:"source_aggregate" | SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. | -| SourceOffline | [*mk20.DataSourceOffline](#datasourceoffline) | json:"source_offline" | SourceOffline defines the data source for offline pieces, including raw size information. | -| SourceHttpPut | [*mk20.DataSourceHttpPut](#datasourcehttpput) | json:"source_httpput" | SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import | - -### Products - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| DDOV1 | [*mk20.DDOV1](#ddov1) | json:"ddo_v1" | DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) | -| RetrievalV1 | [*mk20.RetrievalV1](#retrievalv1) | json:"retrieval_v1" | RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags. | -| PDPV1 | [*mk20.PDPV1](#pdpv1) | json:"pdp_v1" | PDPV1 represents product-specific configuration for PDP version 1 deals. | - -### DBDDOV1 - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| DDO | [*mk20.DDOV1](#ddov1) | json:"ddo" | | -| DealID | [string](https://pkg.go.dev/builtin#string) | json:"deal_id" | | -| Complete | [bool](https://pkg.go.dev/builtin#bool) | json:"complete" | | -| Error | [sql.NullString](https://pkg.go.dev/database/sql#NullString) | json:"error" | | - -### DBPDPV1 - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| PDP | [*mk20.PDPV1](#pdpv1) | json:"pdp" | | -| Complete | [bool](https://pkg.go.dev/builtin#bool) | json:"complete" | | -| Error | [sql.NullString](https://pkg.go.dev/database/sql#NullString) | json:"error" | | - -### DDOV1 - -DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details -for a DDO deal handling. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Provider | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"provider" | Provider specifies the address of the provider | -| PieceManager | [address.Address](https://pkg.go.dev/github.com/filecoin-project/go-address#Address) | json:"piece_manager" | Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs | -| Duration | [abi.ChainEpoch](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#ChainEpoch) | json:"duration" | Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400 | -| AllocationId | [*verifreg.AllocationId](https://pkg.go.dev/github.com/filecoin-project/go-state-types/builtin/v16/verifreg#AllocationId) | json:"allocation_id" | AllocationId represents an aggregated allocation identifier for the deal. | -| ContractAddress | [string](https://pkg.go.dev/builtin#string) | json:"contract_address" | ContractAddress specifies the address of the contract governing the deal | -| ContractVerifyMethod | [string](https://pkg.go.dev/builtin#string) | json:"contract_verify_method" | ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract | -| ContractVerifyMethodParams | [[]byte](https://pkg.go.dev/builtin#byte) | json:"contract_verify_method_params" | ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract | -| NotificationAddress | [string](https://pkg.go.dev/builtin#string) | json:"notification_address" | NotificationAddress specifies the address to which notifications will be relayed to when sector is activated | -| NotificationPayload | [[]byte](https://pkg.go.dev/builtin#byte) | json:"notification_payload" | NotificationPayload holds the notification data typically in a serialized byte array format. | - -### DataSourceAggregate - -DataSourceAggregate represents an aggregated data source containing multiple individual DataSource pieces. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Pieces | [[]mk20.DataSource](#datasource) | json:"pieces" | | - -### DataSourceHTTP - -DataSourceHTTP represents an HTTP-based data source for retrieving piece data, including associated URLs. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| URLs | [[]mk20.HttpUrl](#httpurl) | json:"urls" | URLs lists the HTTP endpoints where the piece data can be fetched. | - -### DataSourceHttpPut - -DataSourceHttpPut represents a data source allowing clients to push piece data after a deal is accepted. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| - -### DataSourceOffline - -DataSourceOffline represents the data source for offline pieces. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| - -### DealStatusResponse - -DealStatusResponse represents the response of a deal's status, including its current state and an optional error message. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| State | [mk20.DealState](#constants-for-dealstate) | json:"status" | State indicates the current processing state of the deal as a DealState value. | -| ErrorMsg | [string](https://pkg.go.dev/builtin#string) | json:"error_msg" | ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. | - -### FormatAggregate - -FormatAggregate represents the aggregated format for piece data, identified by its type. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Type | [mk20.AggregateType](https://pkg.go.dev/github.com/filecoin-project/curio/market/mk20#AggregateType) | json:"type" | Type specifies the type of aggregation for data pieces, represented by an AggregateType value. | -| Sub | [[]mk20.DataSource](#datasource) | json:"sub" | Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format. The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate | - -### FormatBytes - -FormatBytes defines the raw byte representation of data as a format. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| - -### FormatCar - -FormatCar represents the CAR (Content Addressable archive) format for piece data serialization. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| - -### HttpUrl - -HttpUrl represents an HTTP endpoint configuration for fetching piece data. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| URL | [string](https://pkg.go.dev/builtin#string) | json:"url" | URL specifies the HTTP endpoint where the piece data can be fetched. | -| Headers | [http.Header](https://pkg.go.dev/net/http#Header) | json:"headers" | HTTPHeaders represents the HTTP headers associated with the URL. | -| Priority | [int](https://pkg.go.dev/builtin#int) | json:"priority" | Priority indicates the order preference for using the URL in requests, with lower values having higher priority. | -| Fallback | [bool](https://pkg.go.dev/builtin#bool) | json:"fallback" | Fallback indicates whether this URL serves as a fallback option when other URLs fail. | - -### PDPV1 - -PDPV1 represents configuration for product-specific PDP version 1 deals. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| CreateProofSet | [bool](https://pkg.go.dev/builtin#bool) | json:"create_proof_set" | CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider. | -| DeleteProofSet | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_proof_set" | DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. ProofSetID must be defined. | -| AddRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"add_root" | AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined. | -| DeleteRoot | [bool](https://pkg.go.dev/builtin#bool) | json:"delete_root" | DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined. | -| ProofSetID | [*uint64](https://pkg.go.dev/builtin#uint64) | json:"proof_set_id" | ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. | -| ExtraData | [[]byte](https://pkg.go.dev/builtin#byte) | json:"extra_data" | ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. | - -### PieceDataFormat - -PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Car | [*mk20.FormatCar](#formatcar) | json:"car" | Car represents the optional CAR file format, including its metadata and versioning details. | -| Aggregate | [*mk20.FormatAggregate](#formataggregate) | json:"aggregate" | Aggregate holds a reference to the aggregated format of piece data. | -| Raw | [*mk20.FormatBytes](#formatbytes) | json:"raw" | Raw represents the raw format of the piece data, encapsulated as bytes. | - -### PieceInfo - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| PieceCIDV1 | [cid.Cid](https://pkg.go.dev/github.com/ipfs/go-cid#Cid) | json:"piece_cid" | | -| Size | [abi.PaddedPieceSize](https://pkg.go.dev/github.com/filecoin-project/go-state-types/abi#PaddedPieceSize) | json:"size" | | -| RawSize | [uint64](https://pkg.go.dev/builtin#uint64) | json:"raw_size" | | - -### RetrievalV1 - -RetrievalV1 defines a structure for managing retrieval settings - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Indexing | [bool](https://pkg.go.dev/builtin#bool) | json:"indexing" | Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval | -| AnnouncePayload | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_payload" | AnnouncePayload indicates whether the payload should be announced to IPNI. | -| AnnouncePiece | [bool](https://pkg.go.dev/builtin#bool) | json:"announce_piece" | AnnouncePiece indicates whether the piece information should be announced to IPNI. | - -### StartUpload - -StartUpload represents metadata for initiating an upload operation, containing the chunk size of the data to be uploaded. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| ChunkSize | [int64](https://pkg.go.dev/builtin#int64) | json:"chunk_size" | | - -### SupportedContracts - -SupportedContracts represents a collection of contract addresses supported by a system or application. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Contracts | [[]string](https://pkg.go.dev/builtin#string) | json:"contracts" | Contracts represents a list of supported contract addresses in string format. | - -### SupportedDataSources - -SupportedDataSources represents a collection of dats sources supported by the SP. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Sources | [[]string](https://pkg.go.dev/builtin#string) | json:"sources" | Contracts represents a list of supported contract addresses in string format. | - -### SupportedProducts - -SupportedProducts represents a collection of products supported by the SP. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| Products | [[]string](https://pkg.go.dev/builtin#string) | json:"products" | Contracts represents a list of supported contract addresses in string format. | - -### TimeoutReader - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| - -### UploadStatus - -UploadStatus represents the status of a file upload process, including progress and missing chunks. - -| Field | Type | Tag | Description | -|-------|------|-----|-------------| -| TotalChunks | [int](https://pkg.go.dev/builtin#int) | json:"total_chunks" | TotalChunks represents the total number of chunks required for the upload. | -| Uploaded | [int](https://pkg.go.dev/builtin#int) | json:"uploaded" | Uploaded represents the number of chunks successfully uploaded. | -| Missing | [int](https://pkg.go.dev/builtin#int) | json:"missing" | Missing represents the number of chunks that are not yet uploaded. | -| UploadedChunks | [[]int](https://pkg.go.dev/builtin#int) | json:"uploaded_chunks" | UploadedChunks is a slice containing the indices of successfully uploaded chunks. | -| MissingChunks | [[]int](https://pkg.go.dev/builtin#int) | json:"missing_chunks" | MissingChunks is a slice containing the indices of missing chunks. | - -### Constants for ErrorCode - -| Constant | Code | Description | -|----------|------|-------------| -| Ok | 200 | Ok represents a successful operation with an HTTP status code of 200. | -| ErrBadProposal | 400 | ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic. | -| ErrMalformedDataSource | 430 | ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data. | -| ErrUnsupportedDataSource | 422 | ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context. | -| ErrUnsupportedProduct | 423 | ErrUnsupportedProduct indicates that the requested product is not supported by the provider. | -| ErrProductNotEnabled | 424 | ErrProductNotEnabled indicates that the requested product is not enabled on the provider. | -| ErrProductValidationFailed | 425 | ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data. | -| ErrDealRejectedByMarket | 426 | ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules. | -| ErrServiceMaintenance | 503 | ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503. | -| ErrServiceOverloaded | 429 | ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment. | -| ErrMarketNotEnabled | 440 | ErrMarketNotEnabled indicates that the market is not enabled for the requested operation. | -| ErrDurationTooShort | 441 | ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold. | - -### Constants for DealState - -| Constant | Code | Description | -|----------|------|-------------| -| DealStateAccepted | "accepted" | DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system. | -| DealStateProcessing | "processing" | DealStateProcessing represents the state of a deal currently being processed in the pipeline. | -| DealStateSealing | "sealing" | DealStateSealing indicates that the deal is currently being sealed in the system. | -| DealStateIndexing | "indexing" | DealStateIndexing represents the state where a deal is undergoing indexing in the system. | -| DealStateFailed | "failed" | DealStateFailed indicates that the deal has failed due to an error during processing, sealing, or indexing. | -| DealStateComplete | "complete" | DealStateComplete indicates that the deal has successfully completed all processing and is finalized in the system. | - diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json new file mode 100644 index 000000000..9dbf0e813 --- /dev/null +++ b/market/mk20/http/swagger.json @@ -0,0 +1,1060 @@ +{ + "swagger": "2.0", + "info": { + "description": "Curio market APIs", + "title": "Curio Market 2.0 API", + "contact": {} + }, + "paths": { + "/contracts": { + "get": { + "description": "List of supported DDO contracts", + "summary": "List of supported DDO contracts", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/products": { + "get": { + "description": "List of supported products", + "summary": "List of supported products", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/sources": { + "get": { + "description": "List of supported data sources", + "summary": "List of supported dats sources", + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/status/{id}": { + "get": { + "description": "List of supported DDO contracts", + "summary": "List of supported DDO contracts", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK - Success", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/store": { + "post": { + "description": "Make a mk20 deal", + "consumes": [ + "application/json" + ], + "summary": "Make a mk20 deal", + "parameters": [ + { + "description": "mk20.Deal in json format", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/update/{id}": { + "get": { + "description": "Useful for adding adding additional products and updating PoRep duration", + "consumes": [ + "application/json" + ], + "summary": "Update the deal details of existing deals.", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.Deal in json format", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/upload/finalize/{id}": { + "post": { + "description": "Finalizes the upload process once all the chunks are uploaded.", + "consumes": [ + "application/json" + ], + "summary": "Finalizes the upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.deal in json format", + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/upload/{id}": { + "get": { + "description": "Return a json struct detailing the current status of a deal upload.", + "summary": "Status of deal upload", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "UploadStatusCodeOk represents a successful upload operation with status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStatusCodeDealNotFound indicates that the requested deal was not found, corresponding to status code 404", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "425": { + "description": "UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + }, + "500": { + "description": "UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadStatusCode" + } + } + } + }, + "post": { + "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", + "summary": "Starts the upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "UploadStartCodeOk indicates a successful upload start request with status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "409": { + "description": "UploadStartCodeAlreadyStarted indicates that the upload process has already been initiated and cannot be started again", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "500": { + "description": "UploadStartCodeServerError indicates an error occurred on the server while processing an upload start request", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + } + } + } + }, + "/upload/{id}/{chunkNum}": { + "put": { + "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", + "summary": "Upload a file chunk", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "chunkNum", + "name": "chunkNum", + "in": "path", + "required": true + }, + { + "description": "raw binary", + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } + } + ], + "responses": { + "200": { + "description": "UploadOk indicates a successful upload operation, represented by the HTTP status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadNotFound represents an error where the requested upload chunk could not be found, typically corresponding to HTTP status 404", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "409": { + "description": "UploadChunkAlreadyUploaded indicates that the chunk has already been uploaded and cannot be re-uploaded", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "500": { + "description": "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + } + } + } + } + }, + "definitions": { + "address.Address": { + "type": "object" + }, + "cid.Cid": { + "type": "object" + }, + "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { + "type": "integer", + "enum": [ + 0 + ], + "x-enum-varnames": [ + "NoAllocationID" + ] + }, + "http.Header": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "mk20.AggregateType": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "x-enum-varnames": [ + "AggregateTypeNone", + "AggregateTypeV1" + ] + }, + "mk20.DDOV1": { + "type": "object", + "properties": { + "allocation_id": { + "description": "AllocationId represents an aggregated allocation identifier for the deal.", + "allOf": [ + { + "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" + } + ] + }, + "contract_address": { + "description": "ContractAddress specifies the address of the contract governing the deal", + "type": "string" + }, + "contract_verify_method": { + "description": "ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract", + "type": "string" + }, + "contract_verify_method_params": { + "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", + "type": "array", + "items": { + "type": "integer" + } + }, + "duration": { + "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", + "type": "integer" + }, + "notification_address": { + "description": "NotificationAddress specifies the address to which notifications will be relayed to when sector is activated", + "type": "string" + }, + "notification_payload": { + "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", + "type": "array", + "items": { + "type": "integer" + } + }, + "piece_manager": { + "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + }, + "provider": { + "description": "Provider specifies the address of the provider", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + } + } + }, + "mk20.DataSource": { + "type": "object", + "properties": { + "format": { + "description": "Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats.", + "allOf": [ + { + "$ref": "#/definitions/mk20.PieceDataFormat" + } + ] + }, + "piece_cid": { + "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", + "allOf": [ + { + "$ref": "#/definitions/cid.Cid" + } + ] + }, + "source_aggregate": { + "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceAggregate" + } + ] + }, + "source_http": { + "description": "SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceHTTP" + } + ] + }, + "source_httpput": { + "description": "SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceHttpPut" + } + ] + }, + "source_offline": { + "description": "SourceOffline defines the data source for offline pieces, including raw size information.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSourceOffline" + } + ] + } + } + }, + "mk20.DataSourceAggregate": { + "type": "object", + "properties": { + "pieces": { + "type": "array", + "items": { + "$ref": "#/definitions/mk20.DataSource" + } + } + } + }, + "mk20.DataSourceHTTP": { + "type": "object", + "properties": { + "urls": { + "description": "URLs lists the HTTP endpoints where the piece data can be fetched.", + "type": "array", + "items": { + "$ref": "#/definitions/mk20.HttpUrl" + } + } + } + }, + "mk20.DataSourceHttpPut": { + "type": "object" + }, + "mk20.DataSourceOffline": { + "type": "object" + }, + "mk20.Deal": { + "type": "object", + "properties": { + "client": { + "description": "Client wallet for the deal", + "allOf": [ + { + "$ref": "#/definitions/address.Address" + } + ] + }, + "data": { + "description": "Data represents the source of piece data and associated metadata.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DataSource" + } + ] + }, + "identifier": { + "description": "Identifier represents a unique identifier for the deal in UUID format.", + "type": "array", + "items": { + "type": "integer" + } + }, + "products": { + "description": "Products represents a collection of product-specific information associated with a deal", + "allOf": [ + { + "$ref": "#/definitions/mk20.Products" + } + ] + } + } + }, + "mk20.DealCode": { + "type": "integer", + "enum": [ + 200, + 401, + 400, + 404, + 430, + 422, + 423, + 424, + 425, + 426, + 500, + 503, + 429, + 440, + 441 + ], + "x-enum-varnames": [ + "Ok", + "ErrUnAuthorized", + "ErrBadProposal", + "ErrDealNotFound", + "ErrMalformedDataSource", + "ErrUnsupportedDataSource", + "ErrUnsupportedProduct", + "ErrProductNotEnabled", + "ErrProductValidationFailed", + "ErrDealRejectedByMarket", + "ErrServerInternalError", + "ErrServiceMaintenance", + "ErrServiceOverloaded", + "ErrMarketNotEnabled", + "ErrDurationTooShort" + ] + }, + "mk20.FormatAggregate": { + "type": "object", + "properties": { + "sub": { + "description": "Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format.\nThe order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate", + "type": "array", + "items": { + "$ref": "#/definitions/mk20.DataSource" + } + }, + "type": { + "description": "Type specifies the type of aggregation for data pieces, represented by an AggregateType value.", + "allOf": [ + { + "$ref": "#/definitions/mk20.AggregateType" + } + ] + } + } + }, + "mk20.FormatBytes": { + "type": "object" + }, + "mk20.FormatCar": { + "type": "object" + }, + "mk20.HttpUrl": { + "type": "object", + "properties": { + "fallback": { + "description": "Fallback indicates whether this URL serves as a fallback option when other URLs fail.", + "type": "boolean" + }, + "headers": { + "description": "HTTPHeaders represents the HTTP headers associated with the URL.", + "allOf": [ + { + "$ref": "#/definitions/http.Header" + } + ] + }, + "priority": { + "description": "Priority indicates the order preference for using the URL in requests, with lower values having higher priority.", + "type": "integer" + }, + "url": { + "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", + "type": "string" + } + } + }, + "mk20.PDPV1": { + "type": "object", + "properties": { + "add_root": { + "description": "AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined.", + "type": "boolean" + }, + "create_proof_set": { + "description": "CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider.", + "type": "boolean" + }, + "delete_proof_set": { + "description": "DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client.\nProofSetID must be defined.", + "type": "boolean" + }, + "delete_root": { + "description": "DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined.", + "type": "boolean" + }, + "extra_data": { + "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", + "type": "array", + "items": { + "type": "integer" + } + }, + "proof_set_id": { + "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", + "type": "integer" + } + } + }, + "mk20.PieceDataFormat": { + "type": "object", + "properties": { + "aggregate": { + "description": "Aggregate holds a reference to the aggregated format of piece data.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatAggregate" + } + ] + }, + "car": { + "description": "Car represents the optional CAR file format, including its metadata and versioning details.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatCar" + } + ] + }, + "raw": { + "description": "Raw represents the raw format of the piece data, encapsulated as bytes.", + "allOf": [ + { + "$ref": "#/definitions/mk20.FormatBytes" + } + ] + } + } + }, + "mk20.Products": { + "type": "object", + "properties": { + "ddo_v1": { + "description": "DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)", + "allOf": [ + { + "$ref": "#/definitions/mk20.DDOV1" + } + ] + }, + "pdp_v1": { + "description": "PDPV1 represents product-specific configuration for PDP version 1 deals.", + "allOf": [ + { + "$ref": "#/definitions/mk20.PDPV1" + } + ] + }, + "retrieval_v1": { + "description": "RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags.", + "allOf": [ + { + "$ref": "#/definitions/mk20.RetrievalV1" + } + ] + } + } + }, + "mk20.RetrievalV1": { + "type": "object", + "properties": { + "announce_payload": { + "description": "AnnouncePayload indicates whether the payload should be announced to IPNI.", + "type": "boolean" + }, + "announce_piece": { + "description": "AnnouncePiece indicates whether the piece information should be announced to IPNI.", + "type": "boolean" + }, + "indexing": { + "description": "Indexing indicates if the deal is to be indexed in the provider's system to support CIDs based retrieval", + "type": "boolean" + } + } + }, + "mk20.UploadCode": { + "type": "integer", + "enum": [ + 200, + 400, + 404, + 409, + 500 + ], + "x-enum-varnames": [ + "UploadOk", + "UploadBadRequest", + "UploadNotFound", + "UploadChunkAlreadyUploaded", + "UploadServerError" + ] + }, + "mk20.UploadStartCode": { + "type": "integer", + "enum": [ + 200, + 400, + 404, + 409, + 500 + ], + "x-enum-varnames": [ + "UploadStartCodeOk", + "UploadStartCodeBadRequest", + "UploadStartCodeDealNotFound", + "UploadStartCodeAlreadyStarted", + "UploadStartCodeServerError" + ] + }, + "mk20.UploadStatusCode": { + "type": "integer", + "enum": [ + 200, + 404, + 425, + 500 + ], + "x-enum-varnames": [ + "UploadStatusCodeOk", + "UploadStatusCodeDealNotFound", + "UploadStatusCodeUploadNotStarted", + "UploadStatusCodeServerError" + ] + } + }, + "securityDefinitions": { + "CurioAuth": { + "description": "Use the format: `CurioAuth PublicKeyType:PublicKey:Signature`\n\n- `PublicKeyType`: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- `PublicKey`: Base64 string of public key bytes\n- `Signature`: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order.\n- The raw public key bytes (not a human-readable address)\n- The HTTP request path, such as /user/info\n- The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z)\n- These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + } +} \ No newline at end of file diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml new file mode 100644 index 000000000..113d6a758 --- /dev/null +++ b/market/mk20/http/swagger.yaml @@ -0,0 +1,799 @@ +definitions: + address.Address: + type: object + cid.Cid: + type: object + github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId: + enum: + - 0 + type: integer + x-enum-varnames: + - NoAllocationID + http.Header: + additionalProperties: + items: + type: string + type: array + type: object + mk20.AggregateType: + enum: + - 0 + - 1 + type: integer + x-enum-varnames: + - AggregateTypeNone + - AggregateTypeV1 + mk20.DDOV1: + properties: + allocation_id: + allOf: + - $ref: '#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId' + description: AllocationId represents an aggregated allocation identifier for + the deal. + contract_address: + description: ContractAddress specifies the address of the contract governing + the deal + type: string + contract_verify_method: + description: ContractDealIDMethod specifies the method name to verify the + deal and retrieve the deal ID for a contract + type: string + contract_verify_method_params: + description: ContractDealIDMethodParams represents encoded parameters for + the contract verify method if required by the contract + items: + type: integer + type: array + duration: + description: |- + Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. + It must be at least 518400 + type: integer + notification_address: + description: NotificationAddress specifies the address to which notifications + will be relayed to when sector is activated + type: string + notification_payload: + description: NotificationPayload holds the notification data typically in + a serialized byte array format. + items: + type: integer + type: array + piece_manager: + allOf: + - $ref: '#/definitions/address.Address' + description: Actor providing AuthorizeMessage (like f1/f3 wallet) able to + authorize actions such as managing ACLs + provider: + allOf: + - $ref: '#/definitions/address.Address' + description: Provider specifies the address of the provider + type: object + mk20.DataSource: + properties: + format: + allOf: + - $ref: '#/definitions/mk20.PieceDataFormat' + description: Format defines the format of the piece data, which can include + CAR, Aggregate, or Raw formats. + piece_cid: + allOf: + - $ref: '#/definitions/cid.Cid' + description: PieceCID represents the unique identifier (pieceCID V2) for a + piece of data, stored as a CID object. + source_aggregate: + allOf: + - $ref: '#/definitions/mk20.DataSourceAggregate' + description: SourceAggregate represents an aggregated source, comprising multiple + data sources as pieces. + source_http: + allOf: + - $ref: '#/definitions/mk20.DataSourceHTTP' + description: SourceHTTP represents the HTTP-based source of piece data within + a deal, including raw size and URLs for retrieval. + source_httpput: + allOf: + - $ref: '#/definitions/mk20.DataSourceHttpPut' + description: SourceHTTPPut // allow clients to push piece data after deal + accepted, sort of like offline import + source_offline: + allOf: + - $ref: '#/definitions/mk20.DataSourceOffline' + description: SourceOffline defines the data source for offline pieces, including + raw size information. + type: object + mk20.DataSourceAggregate: + properties: + pieces: + items: + $ref: '#/definitions/mk20.DataSource' + type: array + type: object + mk20.DataSourceHTTP: + properties: + urls: + description: URLs lists the HTTP endpoints where the piece data can be fetched. + items: + $ref: '#/definitions/mk20.HttpUrl' + type: array + type: object + mk20.DataSourceHttpPut: + type: object + mk20.DataSourceOffline: + type: object + mk20.Deal: + properties: + client: + allOf: + - $ref: '#/definitions/address.Address' + description: Client wallet for the deal + data: + allOf: + - $ref: '#/definitions/mk20.DataSource' + description: Data represents the source of piece data and associated metadata. + identifier: + description: Identifier represents a unique identifier for the deal in UUID + format. + items: + type: integer + type: array + products: + allOf: + - $ref: '#/definitions/mk20.Products' + description: Products represents a collection of product-specific information + associated with a deal + type: object + mk20.DealCode: + enum: + - 200 + - 401 + - 400 + - 404 + - 430 + - 422 + - 423 + - 424 + - 425 + - 426 + - 500 + - 503 + - 429 + - 440 + - 441 + type: integer + x-enum-varnames: + - Ok + - ErrUnAuthorized + - ErrBadProposal + - ErrDealNotFound + - ErrMalformedDataSource + - ErrUnsupportedDataSource + - ErrUnsupportedProduct + - ErrProductNotEnabled + - ErrProductValidationFailed + - ErrDealRejectedByMarket + - ErrServerInternalError + - ErrServiceMaintenance + - ErrServiceOverloaded + - ErrMarketNotEnabled + - ErrDurationTooShort + mk20.FormatAggregate: + properties: + sub: + description: |- + Sub holds a slice of DataSource, representing details of sub pieces aggregated under this format. + The order must be same as segment index to avoid incorrect indexing of sub pieces in an aggregate + items: + $ref: '#/definitions/mk20.DataSource' + type: array + type: + allOf: + - $ref: '#/definitions/mk20.AggregateType' + description: Type specifies the type of aggregation for data pieces, represented + by an AggregateType value. + type: object + mk20.FormatBytes: + type: object + mk20.FormatCar: + type: object + mk20.HttpUrl: + properties: + fallback: + description: Fallback indicates whether this URL serves as a fallback option + when other URLs fail. + type: boolean + headers: + allOf: + - $ref: '#/definitions/http.Header' + description: HTTPHeaders represents the HTTP headers associated with the URL. + priority: + description: Priority indicates the order preference for using the URL in + requests, with lower values having higher priority. + type: integer + url: + description: URL specifies the HTTP endpoint where the piece data can be fetched. + type: string + type: object + mk20.PDPV1: + properties: + add_root: + description: AddRoot indicated that this deal is meant to add root to a given + ProofSet. ProofSetID must be defined. + type: boolean + create_proof_set: + description: CreateProofSet indicated that this deal is meant to create a + new ProofSet for the client by storage provider. + type: boolean + delete_proof_set: + description: |- + DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. + ProofSetID must be defined. + type: boolean + delete_root: + description: DeleteRoot indicates whether the root of the data should be deleted. + ProofSetID must be defined. + type: boolean + extra_data: + description: ExtraData can be used to send additional information to service + contract when Verifier action like AddRoot, DeleteRoot etc. are performed. + items: + type: integer + type: array + proof_set_id: + description: ProofSetID is PDP verified contract proofset ID. It must be defined + for all deals except when CreateProofSet is true. + type: integer + type: object + mk20.PieceDataFormat: + properties: + aggregate: + allOf: + - $ref: '#/definitions/mk20.FormatAggregate' + description: Aggregate holds a reference to the aggregated format of piece + data. + car: + allOf: + - $ref: '#/definitions/mk20.FormatCar' + description: Car represents the optional CAR file format, including its metadata + and versioning details. + raw: + allOf: + - $ref: '#/definitions/mk20.FormatBytes' + description: Raw represents the raw format of the piece data, encapsulated + as bytes. + type: object + mk20.Products: + properties: + ddo_v1: + allOf: + - $ref: '#/definitions/mk20.DDOV1' + description: DDOV1 represents a product v1 configuration for Direct Data Onboarding + (DDO) + pdp_v1: + allOf: + - $ref: '#/definitions/mk20.PDPV1' + description: PDPV1 represents product-specific configuration for PDP version + 1 deals. + retrieval_v1: + allOf: + - $ref: '#/definitions/mk20.RetrievalV1' + description: RetrievalV1 represents configuration for retrieval settings in + the system, including indexing and announcement flags. + type: object + mk20.RetrievalV1: + properties: + announce_payload: + description: AnnouncePayload indicates whether the payload should be announced + to IPNI. + type: boolean + announce_piece: + description: AnnouncePiece indicates whether the piece information should + be announced to IPNI. + type: boolean + indexing: + description: Indexing indicates if the deal is to be indexed in the provider's + system to support CIDs based retrieval + type: boolean + type: object + mk20.UploadCode: + enum: + - 200 + - 400 + - 404 + - 409 + - 500 + type: integer + x-enum-varnames: + - UploadOk + - UploadBadRequest + - UploadNotFound + - UploadChunkAlreadyUploaded + - UploadServerError + mk20.UploadStartCode: + enum: + - 200 + - 400 + - 404 + - 409 + - 500 + type: integer + x-enum-varnames: + - UploadStartCodeOk + - UploadStartCodeBadRequest + - UploadStartCodeDealNotFound + - UploadStartCodeAlreadyStarted + - UploadStartCodeServerError + mk20.UploadStatusCode: + enum: + - 200 + - 404 + - 425 + - 500 + type: integer + x-enum-varnames: + - UploadStatusCodeOk + - UploadStatusCodeDealNotFound + - UploadStatusCodeUploadNotStarted + - UploadStatusCodeServerError +info: + contact: {} + description: Curio market APIs + title: Curio Market 2.0 API +paths: + /contracts: + get: + description: List of supported DDO contracts + responses: + "200": + description: OK - Success + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List of supported DDO contracts + /products: + get: + description: List of supported products + responses: + "200": + description: OK - Success + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List of supported products + /sources: + get: + description: List of supported data sources + responses: + "200": + description: OK - Success + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List of supported dats sources + /status/{id}: + get: + description: List of supported DDO contracts + parameters: + - description: id + in: path + name: id + required: true + type: string + responses: + "200": + description: OK - Success + schema: + type: string + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List of supported DDO contracts + /store: + post: + consumes: + - application/json + description: Make a mk20 deal + parameters: + - description: mk20.Deal in json format + in: body + name: body + required: true + schema: + $ref: '#/definitions/mk20.Deal' + responses: + "200": + description: Ok represents a successful operation with an HTTP status code + of 200 + schema: + $ref: '#/definitions/mk20.DealCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: ErrDealNotFound indicates that the specified deal could not + be found, corresponding to the HTTP status code 404 + schema: + $ref: '#/definitions/mk20.DealCode' + "422": + description: ErrUnsupportedDataSource indicates the specified data source + is not supported or disabled for use in the current context + schema: + $ref: '#/definitions/mk20.DealCode' + "423": + description: ErrUnsupportedProduct indicates that the requested product + is not supported by the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "424": + description: ErrProductNotEnabled indicates that the requested product is + not enabled on the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "425": + description: ErrProductValidationFailed indicates a failure during product-specific + validation due to invalid or missing data + schema: + $ref: '#/definitions/mk20.DealCode' + "426": + description: ErrDealRejectedByMarket indicates that a proposed deal was + rejected by the market for not meeting its acceptance criteria or rules + schema: + $ref: '#/definitions/mk20.DealCode' + "429": + description: ErrServiceOverloaded indicates that the service is overloaded + and cannot process the request at the moment + schema: + $ref: '#/definitions/mk20.DealCode' + "430": + description: ErrMalformedDataSource indicates that the provided data source + is incorrectly formatted or contains invalid data + schema: + $ref: '#/definitions/mk20.DealCode' + "440": + description: ErrMarketNotEnabled indicates that the market is not enabled + for the requested operation + schema: + $ref: '#/definitions/mk20.DealCode' + "441": + description: ErrDurationTooShort indicates that the provided duration value + does not meet the minimum required threshold + schema: + $ref: '#/definitions/mk20.DealCode' + "500": + description: ErrServerInternalError indicates an internal server error with + a corresponding error code of 500 + schema: + $ref: '#/definitions/mk20.DealCode' + "503": + description: ErrServiceMaintenance indicates that the service is temporarily + unavailable due to maintenance, corresponding to HTTP status code 503 + schema: + $ref: '#/definitions/mk20.DealCode' + summary: Make a mk20 deal + /update/{id}: + get: + consumes: + - application/json + description: Useful for adding adding additional products and updating PoRep + duration + parameters: + - description: id + in: path + name: id + required: true + type: string + - description: mk20.Deal in json format + in: body + name: body + required: true + schema: + $ref: '#/definitions/mk20.Deal' + responses: + "200": + description: Ok represents a successful operation with an HTTP status code + of 200 + schema: + $ref: '#/definitions/mk20.DealCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: ErrDealNotFound indicates that the specified deal could not + be found, corresponding to the HTTP status code 404 + schema: + $ref: '#/definitions/mk20.DealCode' + "422": + description: ErrUnsupportedDataSource indicates the specified data source + is not supported or disabled for use in the current context + schema: + $ref: '#/definitions/mk20.DealCode' + "423": + description: ErrUnsupportedProduct indicates that the requested product + is not supported by the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "424": + description: ErrProductNotEnabled indicates that the requested product is + not enabled on the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "425": + description: ErrProductValidationFailed indicates a failure during product-specific + validation due to invalid or missing data + schema: + $ref: '#/definitions/mk20.DealCode' + "426": + description: ErrDealRejectedByMarket indicates that a proposed deal was + rejected by the market for not meeting its acceptance criteria or rules + schema: + $ref: '#/definitions/mk20.DealCode' + "429": + description: ErrServiceOverloaded indicates that the service is overloaded + and cannot process the request at the moment + schema: + $ref: '#/definitions/mk20.DealCode' + "430": + description: ErrMalformedDataSource indicates that the provided data source + is incorrectly formatted or contains invalid data + schema: + $ref: '#/definitions/mk20.DealCode' + "440": + description: ErrMarketNotEnabled indicates that the market is not enabled + for the requested operation + schema: + $ref: '#/definitions/mk20.DealCode' + "441": + description: ErrDurationTooShort indicates that the provided duration value + does not meet the minimum required threshold + schema: + $ref: '#/definitions/mk20.DealCode' + "500": + description: ErrServerInternalError indicates an internal server error with + a corresponding error code of 500 + schema: + $ref: '#/definitions/mk20.DealCode' + "503": + description: ErrServiceMaintenance indicates that the service is temporarily + unavailable due to maintenance, corresponding to HTTP status code 503 + schema: + $ref: '#/definitions/mk20.DealCode' + summary: Update the deal details of existing deals. + /upload/{id}: + get: + description: Return a json struct detailing the current status of a deal upload. + parameters: + - description: id + in: path + name: id + required: true + type: string + responses: + "200": + description: UploadStatusCodeOk represents a successful upload operation + with status code 200 + schema: + $ref: '#/definitions/mk20.UploadStatusCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: UploadStatusCodeDealNotFound indicates that the requested deal + was not found, corresponding to status code 404 + schema: + $ref: '#/definitions/mk20.UploadStatusCode' + "425": + description: UploadStatusCodeUploadNotStarted indicates that the upload + process has not started yet + schema: + $ref: '#/definitions/mk20.UploadStatusCode' + "500": + description: UploadStatusCodeServerError indicates an internal server error + occurred during the upload process, corresponding to status code 500 + schema: + $ref: '#/definitions/mk20.UploadStatusCode' + summary: Status of deal upload + post: + description: Initializes the upload for a deal. Each upload must be initialized + before chunks can be uploaded for a deal. + parameters: + - description: id + in: path + name: id + required: true + type: string + responses: + "200": + description: UploadStartCodeOk indicates a successful upload start request + with status code 200 + schema: + $ref: '#/definitions/mk20.UploadStartCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: UploadStartCodeDealNotFound represents a 404 status indicating + the deal was not found during the upload start process + schema: + $ref: '#/definitions/mk20.UploadStartCode' + "409": + description: UploadStartCodeAlreadyStarted indicates that the upload process + has already been initiated and cannot be started again + schema: + $ref: '#/definitions/mk20.UploadStartCode' + "500": + description: UploadStartCodeServerError indicates an error occurred on the + server while processing an upload start request + schema: + $ref: '#/definitions/mk20.UploadStartCode' + summary: Starts the upload process + /upload/{id}/{chunkNum}: + put: + description: Allows uploading chunks for a deal file. Method can be called in + parallel to speed up uploads. + parameters: + - description: id + in: path + name: id + required: true + type: string + - description: chunkNum + in: path + name: chunkNum + required: true + type: string + - description: raw binary + in: body + name: body + required: true + schema: + items: + type: integer + type: array + responses: + "200": + description: UploadOk indicates a successful upload operation, represented + by the HTTP status code 200 + schema: + $ref: '#/definitions/mk20.UploadCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: UploadNotFound represents an error where the requested upload + chunk could not be found, typically corresponding to HTTP status 404 + schema: + $ref: '#/definitions/mk20.UploadCode' + "409": + description: UploadChunkAlreadyUploaded indicates that the chunk has already + been uploaded and cannot be re-uploaded + schema: + $ref: '#/definitions/mk20.UploadCode' + "500": + description: UploadServerError indicates a server-side error occurred during + the upload process, represented by the HTTP status code 500 + schema: + $ref: '#/definitions/mk20.UploadCode' + summary: Upload a file chunk + /upload/finalize/{id}: + post: + consumes: + - application/json + description: Finalizes the upload process once all the chunks are uploaded. + parameters: + - description: id + in: path + name: id + required: true + type: string + - description: mk20.deal in json format + in: body + name: body + schema: + $ref: '#/definitions/mk20.Deal' + responses: + "200": + description: Ok represents a successful operation with an HTTP status code + of 200 + schema: + $ref: '#/definitions/mk20.DealCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: ErrDealNotFound indicates that the specified deal could not + be found, corresponding to the HTTP status code 404 + schema: + $ref: '#/definitions/mk20.DealCode' + "422": + description: ErrUnsupportedDataSource indicates the specified data source + is not supported or disabled for use in the current context + schema: + $ref: '#/definitions/mk20.DealCode' + "423": + description: ErrUnsupportedProduct indicates that the requested product + is not supported by the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "424": + description: ErrProductNotEnabled indicates that the requested product is + not enabled on the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "425": + description: ErrProductValidationFailed indicates a failure during product-specific + validation due to invalid or missing data + schema: + $ref: '#/definitions/mk20.DealCode' + "426": + description: ErrDealRejectedByMarket indicates that a proposed deal was + rejected by the market for not meeting its acceptance criteria or rules + schema: + $ref: '#/definitions/mk20.DealCode' + "429": + description: ErrServiceOverloaded indicates that the service is overloaded + and cannot process the request at the moment + schema: + $ref: '#/definitions/mk20.DealCode' + "430": + description: ErrMalformedDataSource indicates that the provided data source + is incorrectly formatted or contains invalid data + schema: + $ref: '#/definitions/mk20.DealCode' + "440": + description: ErrMarketNotEnabled indicates that the market is not enabled + for the requested operation + schema: + $ref: '#/definitions/mk20.DealCode' + "441": + description: ErrDurationTooShort indicates that the provided duration value + does not meet the minimum required threshold + schema: + $ref: '#/definitions/mk20.DealCode' + "500": + description: ErrServerInternalError indicates an internal server error with + a corresponding error code of 500 + schema: + $ref: '#/definitions/mk20.DealCode' + "503": + description: ErrServiceMaintenance indicates that the service is temporarily + unavailable due to maintenance, corresponding to HTTP status code 503 + schema: + $ref: '#/definitions/mk20.DealCode' + summary: Finalizes the upload process +securityDefinitions: + CurioAuth: + description: |- + Use the format: `CurioAuth PublicKeyType:PublicKey:Signature` + + - `PublicKeyType`: String representation of type of wallet (e.g., "ed25519", "bls", "secp256k1") + - `PublicKey`: Base64 string of public key bytes + - `Signature`: Signature is Base64 string of signature bytes. + - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order. + - The raw public key bytes (not a human-readable address) + - The HTTP request path, such as /user/info + - The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z) + - These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. + in: header + name: Authorization + type: apiKey +swagger: "2.0" diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 66591a9da..cf0cb3c76 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -45,7 +45,7 @@ type MK20API interface { type MK20 struct { miners []address.Address - db *harmonydb.DB + DB *harmonydb.DB api MK20API ethClient *ethclient.Client si paths.SectorIndex @@ -54,6 +54,7 @@ type MK20 struct { as *multictladdr.MultiAddressSelector sc *ffi.SealCalls maxParallelUploads *atomic.Int64 + unknowClient bool } func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorIndex, mapi MK20API, ethClient *ethclient.Client, cfg *config.CurioConfig, as *multictladdr.MultiAddressSelector, sc *ffi.SealCalls) (*MK20, error) { @@ -84,7 +85,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI return &MK20{ miners: miners, - db: db, + DB: db, api: mapi, ethClient: ethClient, si: si, @@ -93,9 +94,15 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI as: as, sc: sc, maxParallelUploads: new(atomic.Int64), + unknowClient: !cfg.Market.StorageMarketConfig.MK20.DenyUnknownClients, }, nil } +// ExecuteDeal take a *Deal and returns ProviderDealRejectionInfo which has ErrorCode and Reason +// @param deal *Deal +// @Return DealCode +// @Return Reason string + func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { defer func() { if r := recover(); r != nil { @@ -107,13 +114,13 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti }() // Validate the DataSource - code, err := deal.Validate(m.db, &m.cfg.Market.StorageMarketConfig.MK20) + code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ - HTTPCode: int(code), + HTTPCode: code, } - if code == http.StatusInternalServerError { + if code == ErrServerInternalError { ret.Reason = "Internal server error" } else { ret.Reason = err.Error() @@ -143,13 +150,13 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje return rejection } - id, code, err := deal.Products.DDOV1.GetDealID(ctx, m.db, m.ethClient) + id, code, err := deal.Products.DDOV1.GetDealID(ctx, m.DB, m.ethClient) if err != nil { log.Errorw("error getting deal ID", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ - HTTPCode: int(code), + HTTPCode: code, } - if code == http.StatusInternalServerError { + if code == ErrServerInternalError { ret.Reason = "Internal server error" } else { ret.Reason = err.Error() @@ -161,7 +168,7 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje // TODO: Backpressure, client filter - comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { err = deal.SaveToDB(tx) if err != nil { return false, err @@ -175,10 +182,8 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje if n != 1 { return false, fmt.Errorf("expected 1 row to be updated, got %d", n) } - if deal.Data.SourceHttpPut != nil { - _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id, waiting_for_data) VALUES ($1, TRUE) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) - } else { + if deal.Data.SourceHttpPut == nil { _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) } @@ -191,14 +196,14 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje if err != nil { log.Errorw("error inserting deal into DB", "deal", deal, "error", err) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, } } if !comm { log.Errorw("error committing deal into DB", "deal", deal) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, } } @@ -212,23 +217,30 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { if !lo.Contains(m.miners, deal.Products.DDOV1.Provider) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Provider not available in Curio cluster", }, nil } + if deal.Data != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Data Source must be defined for a DDO deal", + }, nil + } + size, err := deal.Size() if err != nil { log.Errorw("error getting deal size", "deal", deal, "error", err) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Error getting deal size from PieceCID", }, nil } if size > abi.PaddedPieceSize(m.sm[deal.Products.DDOV1.Provider]) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Deal size is larger than the miner's sector size", }, nil } @@ -237,7 +249,7 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe if deal.Products.RetrievalV1 != nil { if deal.Products.RetrievalV1.Indexing { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Raw bytes deal cannot be indexed", }, nil } @@ -247,7 +259,7 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe if deal.Products.DDOV1.AllocationId != nil { if size < abi.PaddedPieceSize(verifreg.MinimumVerifiedAllocationSize) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Verified piece size must be at least 1MB", }, nil } @@ -255,13 +267,13 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe alloc, err := m.api.StateGetAllocation(ctx, deal.Client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) if err != nil { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, }, xerrors.Errorf("getting allocation: %w", err) } if alloc == nil { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Verified piece must have a valid allocation ID", }, nil } @@ -269,14 +281,14 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe clientID, err := address.IDFromAddress(deal.Client) if err != nil { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Invalid client address", }, nil } if alloc.Client != abi.ActorID(clientID) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "client address does not match the allocation client address", }, nil } @@ -284,34 +296,34 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe prov, err := address.NewIDAddress(uint64(alloc.Provider)) if err != nil { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, }, xerrors.Errorf("getting provider address: %w", err) } if !lo.Contains(m.miners, prov) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Allocation provider does not belong to the list of miners in Curio cluster", }, nil } if !deal.Data.PieceCID.Equals(alloc.Data) { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Allocation data CID does not match the piece CID", }, nil } if size != alloc.Size { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Allocation size does not match the piece size", }, nil } if alloc.TermMin > miner.MaxSectorExpirationExtension-policy.SealRandomnessLookback { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Allocation term min is greater than the maximum sector expiration extension", }, nil } @@ -334,7 +346,7 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } // Save deal to DB and start pipeline if required - comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { // Save deal err = deal.SaveToDB(tx) if err != nil { @@ -356,13 +368,13 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje if err != nil { log.Errorw("error inserting PDP deal into DB", "deal", deal, "error", err) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, } } if !comm { log.Errorw("error committing PDP deal into DB", "deal", deal) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusInternalServerError, + HTTPCode: ErrServerInternalError, } } log.Debugw("PDP deal inserted in DB", "deal", deal.Identifier.String()) @@ -653,39 +665,65 @@ func markDownloaded(ctx context.Context, db *harmonydb.DB) { } } +// UpdateDeal updates the details of a deal specified by its ID and writes the result or error to the provided HTTP response writer. +// @param id ulid.ULID +// @param deal *Deal +// @Return DealCode + func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, w http.ResponseWriter) { + if deal == nil { + http.Error(w, "deal not defined", int(ErrBadProposal)) + return + } + ctx := context.Background() + + allowed, err := AuthenticateClient(m.DB, deal.Identifier.String(), deal.Client.String()) + if err != nil { + log.Errorw("deal rejected", "deal", deal, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if !allowed { + log.Errorw("deal rejected as client is not authorized", "deal", deal) + http.Error(w, "client not authorized", int(ErrUnAuthorized)) + return + } + var exists bool - err := m.db.QueryRow(ctx, `SELECT EXISTS ( + err = m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 FROM market_mk20_deal WHERE id = $1)`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if deal exists", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(ErrServerInternalError)) return } if !exists { - http.Error(w, "", http.StatusNotFound) + http.Error(w, "", int(ErrDealNotFound)) return } - if deal == nil { - http.Error(w, "deal not defined", int(ErrBadProposal)) - } - - code, err := m.updateDealDetails(id, deal) + code, nd, np, err := m.updateDealDetails(id, deal) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) - if code == http.StatusInternalServerError { - http.Error(w, "", http.StatusInternalServerError) + if code == ErrServerInternalError { + http.Error(w, "", int(ErrServerInternalError)) } else { http.Error(w, err.Error(), int(code)) } return } + // Initiate new pipelines for DDO if required + for _, p := range np { + if p == ProductNameDDOV1 { + m.processDDODeal(ctx, nd) + } + } + w.WriteHeader(http.StatusOK) } diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index b238e7091..c206e0a7c 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -22,26 +22,29 @@ import ( "github.com/filecoin-project/curio/lib/storiface" ) +// HandleUploadStatus retrieves and returns the upload status of a deal, including chunk completion details, or reports errors if the process fails. +// @param ID ulid.ULID +// @Return UploadStatusCode + func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.ResponseWriter) { var exists bool - err := m.db.QueryRow(ctx, `SELECT EXISTS ( + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 - FROM market_mk20_pipeline_waiting - WHERE id = $1 AND waiting_for_data = TRUE - )`, id.String()).Scan(&exists) + FROM market_mk20_upload_waiting + WHERE id = $1;)`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if upload is waiting for data", "deal", id, "error", err) - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(int(UploadStatusCodeServerError)) return } if !exists { - http.Error(w, "deal not found", http.StatusNotFound) + http.Error(w, "deal not found", int(UploadStatusCodeDealNotFound)) return } var ret UploadStatus - err = m.db.QueryRow(ctx, `SELECT + err = m.DB.QueryRow(ctx, `SELECT COUNT(*) AS total, COUNT(*) FILTER (WHERE complete) AS complete, COUNT(*) FILTER (WHERE NOT complete) AS missing, @@ -56,23 +59,23 @@ func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.Resp if err != nil { if !errors.Is(err, pgx.ErrNoRows) { log.Errorw("failed to get upload status", "deal", id, "error", err) - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(int(UploadStatusCodeServerError)) return } - http.Error(w, "chunk size not updated", http.StatusNotFound) + http.Error(w, "upload not initiated", int(UploadStatusCodeUploadNotStarted)) return } data, err := json.Marshal(ret) if err != nil { log.Errorw("failed to marshal upload status", "deal", id, "error", err) - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(int(UploadStatusCodeServerError)) return } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) + w.WriteHeader(int(UploadStatusCodeOk)) _, err = w.Write(data) if err != nil { @@ -80,85 +83,105 @@ func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.Resp } } -func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, chunkSize int64, w http.ResponseWriter) { +// HandleUploadStart handles the initialization of a file upload process for a specific deal, validating input and creating database entries. +// @param ID ulid.ULID +// @param upload StartUpload +// @Return UploadStartCode + +func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, upload StartUpload, w http.ResponseWriter) { + chunkSize := upload.ChunkSize + if upload.RawSize == 0 { + log.Errorw("raw size must be greater than 0", "id", id) + http.Error(w, "raw size must be greater than 0", int(UploadStartCodeBadRequest)) + return + } + if chunkSize == 0 { log.Errorw("chunk size must be greater than 0", "id", id) - http.Error(w, "chunk size must be greater than 0", http.StatusBadRequest) + http.Error(w, "chunk size must be greater than 0", int(UploadStartCodeBadRequest)) return } // Check if chunk size is a power of 2 if chunkSize&(chunkSize-1) != 0 { log.Errorw("chunk size must be a power of 2", "id", id) - http.Error(w, "chunk size must be a power of 2", http.StatusBadRequest) + http.Error(w, "chunk size must be a power of 2", int(UploadStartCodeBadRequest)) return } // Check that chunk size align with config if chunkSize < m.cfg.Market.StorageMarketConfig.MK20.MinimumChunkSize { log.Errorw("chunk size too small", "id", id) - http.Error(w, "chunk size too small", http.StatusBadRequest) + http.Error(w, "chunk size too small", int(UploadStartCodeBadRequest)) return } if chunkSize > m.cfg.Market.StorageMarketConfig.MK20.MaximumChunkSize { log.Errorw("chunk size too large", "id", id) - http.Error(w, "chunk size too large", http.StatusBadRequest) + http.Error(w, "chunk size too large", int(UploadStartCodeBadRequest)) return } // Check if deal exists var exists bool - err := m.db.QueryRow(ctx, `SELECT EXISTS ( + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 - FROM market_mk20_deal + FROM market_mk20_upload_waiting WHERE id = $1 );`, id.String()).Scan(&exists) if err != nil { - log.Errorw("failed to check if deal exists", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + log.Errorw("failed to check if deal is waiting for upload to start", "deal", id, "error", err) + http.Error(w, "", int(UploadStartCodeServerError)) return } if !exists { - http.Error(w, "deal not found", http.StatusNotFound) + http.Error(w, "deal not found", int(UploadStartCodeDealNotFound)) return } // Check if we already started the upload var started bool - err = m.db.QueryRow(ctx, `SELECT EXISTS ( + err = m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 FROM market_mk20_deal_chunk WHERE id = $1);`, id.String()).Scan(&started) if err != nil { log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadStartCodeServerError)) return } if started { - http.Error(w, "deal upload has already started", http.StatusTooManyRequests) + http.Error(w, "deal upload has already started", int(UploadStartCodeAlreadyStarted)) return } - deal, err := DealFromDB(ctx, m.db, id) + deal, err := DealFromDB(ctx, m.DB, id) if err != nil { log.Errorw("failed to get deal from db", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadStartCodeServerError)) return } - rawSize, err := deal.RawSize() - if err != nil { - log.Errorw("failed to get raw size of deal", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) - return + var rawSize uint64 + + if deal.Data != nil { + rawSize, err = deal.RawSize() + if err != nil { + log.Errorw("failed to get raw size of deal", "deal", id, "error", err) + http.Error(w, "", int(UploadStartCodeServerError)) + return + } + if rawSize != upload.RawSize { + log.Errorw("raw size of deal does not match the one provided in deal", "deal", id, "error", err) + http.Error(w, "", int(UploadStartCodeBadRequest)) + } } numChunks := int(math.Ceil(float64(rawSize) / float64(chunkSize))) // Create rows in market_mk20_deal_chunk for each chunk for the ID - comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { batch := &pgx.Batch{} batchSize := 15000 for i := 1; i <= numChunks; i++ { @@ -191,23 +214,31 @@ func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, chunkSize in }, harmonydb.OptionRetry()) if err != nil { log.Errorw("failed to create chunks for deal", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadStartCodeServerError)) return } if !comm { log.Errorw("failed to create chunks for deal", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadStartCodeServerError)) return } - w.WriteHeader(http.StatusOK) + w.WriteHeader(int(UploadStartCodeOk)) } +// HandleUploadChunk processes a single chunk upload for a deal and validates its state. +// It checks if the chunk exists, ensures it's not already uploaded, and stores it if valid. +// The function updates the database with chunk details and manages transaction rolls back on failure. +// @param id ulid.ULID +// @param chunk int +// @param data []byte +// @Return UploadCode + func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w http.ResponseWriter) { ctx := context.Background() defer data.Close() if chunk < 1 { - http.Error(w, "chunk must be greater than 0", http.StatusBadRequest) + http.Error(w, "chunk must be greater than 0", int(UploadBadRequest)) return } @@ -217,32 +248,32 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w Complete bool `db:"complete"` RefID sql.NullInt64 `db:"ref_id"` } - err := m.db.Select(ctx, &chunkDetails, `SELECT chunk, chunk_size, ref_id, complete + err := m.DB.Select(ctx, &chunkDetails, `SELECT chunk, chunk_size, ref_id, complete FROM market_mk20_deal_chunk WHERE id = $1 AND chunk = $2`, id.String(), chunk) if err != nil { log.Errorw("failed to check if chunk exists", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) } if len(chunkDetails) == 0 { - http.Error(w, "chunk not found", http.StatusNotFound) + http.Error(w, "chunk not found", int(UploadNotFound)) return } if len(chunkDetails) > 1 { log.Errorw("chunk exists multiple times", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } if chunkDetails[0].Complete { - http.Error(w, "chunk already uploaded", http.StatusConflict) + http.Error(w, "chunk already uploaded", int(UploadChunkAlreadyUploaded)) return } if chunkDetails[0].RefID.Valid { - http.Error(w, "chunk already uploaded", http.StatusConflict) + http.Error(w, "chunk already uploaded", int(UploadChunkAlreadyUploaded)) return } @@ -261,7 +292,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w n, err := wr.Write([]byte(fmt.Sprintf("%s, %d, %d, %s", id.String(), chunk, chunkSize, time.Now().String()))) if err != nil { log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } digest, tsize, err := wr.Digest() @@ -273,7 +304,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w var pnum, refID int64 // Generate piece park details with tmp pieceCID and Size - comm, err := m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 @@ -306,13 +337,13 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w if err != nil { log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } if !comm { log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", "failed to commit transaction") - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } @@ -321,7 +352,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w failed := true defer func() { if failed { - _, err = m.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + _, err = m.DB.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) if err != nil { log.Errorw("failed to delete parked piece ref", "deal", id, "chunk", chunk, "error", err) } @@ -332,14 +363,14 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w pi, err := m.sc.WriteUploadPiece(ctx, storiface.PieceNumber(pnum), chunkSize, reader, storiface.PathSealing) if err != nil { log.Errorw("failed to write piece", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } log.Debugw("piece stored", "deal", id, "chunk", chunk) // Update piece park details with correct values - comm, err = m.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + comm, err = m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { n, err := tx.Exec(`UPDATE parked_pieces SET piece_cid = $1, piece_padded_size = $2, @@ -370,33 +401,50 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w if err != nil { log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } if !comm { log.Errorw("failed to update chunk", "deal", id, "chunk", chunk, "error", "failed to commit transaction") - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(UploadServerError)) return } log.Debugw("chunk upload finished", "deal", id, "chunk", chunk) failed = false - + w.WriteHeader(int(UploadOk)) } +// HandleUploadFinalize completes the upload process for a deal by verifying its chunks, updating the deal, and marking the upload as finalized. +// @param id ulid.ULID +// @param deal *Deal [optional] +// @Return DealCode + func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { ctx := context.Background() + if deal != nil { + allow, err := AuthenticateClient(m.DB, id.String(), deal.Client.String()) + if err != nil { + log.Errorw("failed to authenticate client", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if !allow { + http.Error(w, "client is not authorized to finalize deal", http.StatusUnauthorized) + return + } + } var exists bool - err := m.db.QueryRow(ctx, `SELECT EXISTS ( + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 FROM market_mk20_deal_chunk WHERE id = $1 AND complete = FALSE OR complete IS NULL )`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(ErrServerInternalError)) return } @@ -407,71 +455,97 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri if deal != nil { // This is a deal where DataSource was not set - we should update the deal - code, err := m.updateDealDetails(id, deal) + code, ndeal, _, err := m.updateDealDetails(id, deal) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) - if code == http.StatusInternalServerError { - http.Error(w, "", http.StatusInternalServerError) + if code == ErrServerInternalError { + http.Error(w, "", int(ErrServerInternalError)) } else { http.Error(w, err.Error(), int(code)) } return } + rawSize, err := ndeal.RawSize() + if err != nil { + log.Errorw("failed to get raw size of deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + + var valid bool + + err = m.DB.QueryRow(ctx, `SELECT SUM(chunk_size) = $2 AS valid + FROM market_mk20_deal_chunk + WHERE id = $1;`, id.String(), rawSize).Scan(&valid) + if err != nil { + log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if !valid { + log.Errorw("deal upload finalize failed", "deal", id, "error", "deal raw size does not match the sum of chunks") + http.Error(w, "deal raw size does not match the sum of chunks", int(ErrBadProposal)) + return + } } // Now update the upload status to trigger the correct pipeline - n, err := m.db.Exec(ctx, `UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) + n, err := m.DB.Exec(ctx, `UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) if err != nil { log.Errorw("failed to finalize deal upload", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(ErrServerInternalError)) return } if n == 0 { log.Errorw("failed to finalize deal upload", "deal", id, "error", err) - http.Error(w, "", http.StatusInternalServerError) + http.Error(w, "", int(ErrServerInternalError)) return } w.WriteHeader(http.StatusOK) } -func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (ErrorCode, error) { +func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []ProductName, error) { ctx := context.Background() // Let's not use request context to avoid DB inconsistencies if deal.Identifier.Compare(id) != 0 { - return ErrBadProposal, xerrors.Errorf("deal ID and proposal ID do not match") + return ErrBadProposal, nil, nil, xerrors.Errorf("deal ID and proposal ID do not match") + } + + if deal.Data == nil { + return ErrBadProposal, nil, nil, xerrors.Errorf("deal data is nil") } // Validate the deal - code, err := deal.Validate(m.db, &m.cfg.Market.StorageMarketConfig.MK20) + code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20) if err != nil { - return code, err + return code, nil, nil, err } log.Debugw("deal validated", "deal", deal.Identifier.String()) // Verify we have a deal is DB var exists bool - err = m.db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_deal WHERE id = $1)`, id.String()).Scan(&exists) + err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_deal WHERE id = $1)`, id.String()).Scan(&exists) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("failed to check if deal exists: %w", err) + return ErrServerInternalError, nil, nil, xerrors.Errorf("failed to check if deal exists: %w", err) } if !exists { - return http.StatusNotFound, xerrors.Errorf("deal not found") + return ErrDealNotFound, nil, nil, xerrors.Errorf("deal not found") } // Get updated deal - ndeal, code, err := UpdateDealDetails(ctx, m.db, id, deal, &m.cfg.Market.StorageMarketConfig.MK20) + ndeal, code, np, err := UpdateDealDetails(ctx, m.DB, id, deal, &m.cfg.Market.StorageMarketConfig.MK20) if err != nil { - return code, err + return code, nil, nil, err } // Save the updated deal to DB - err = ndeal.UpdateDeal(ctx, m.db) + err = ndeal.UpdateDeal(ctx, m.DB) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("failed to update deal: %w", err) + return ErrServerInternalError, nil, nil, xerrors.Errorf("failed to update deal: %w", err) } - return http.StatusOK, nil + return Ok, ndeal, np, nil } diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index eb06c1a9a..f6108dfeb 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -13,12 +13,25 @@ import ( "github.com/yugabyte/pgx/v5" ) +// DealStatus retrieves the status of a specific deal by querying the database and determining the current state for both PDP and DDO processing. +// @param id [ulid.ULID] +// @Return http.StatusNotFound +// @Return http.StatusInternalServerError +// @Return *DealProductStatusResponse + func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { // Check if we ever accepted this deal - var dealError sql.NullString + var pdp_complete, ddo_complete sql.NullBool + var pdp_error, ddo_error sql.NullString - err := m.db.QueryRow(ctx, `SELECT error FROM market_mk20_deal WHERE id = $1;`, id.String()).Scan(&dealError) + err := m.DB.QueryRow(ctx, `SELECT + pdp_v1->>'complete' AS pdp_complete, + pdp_v1->>'error' AS pdp_error, + ddo_v1->>'complete' AS ddo_complete, + ddo_v1->>'error' AS ddo_error + FROM market_mk20_deal + WHERE id = $1;`, id.String()).Scan(&pdp_complete, &pdp_error, &ddo_complete, &ddo_error) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return &DealStatus{ @@ -30,18 +43,35 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { HTTPCode: http.StatusInternalServerError, } } - if dealError.Valid { + // Handle corner case if now product rows + if !pdp_complete.Valid && !ddo_complete.Valid { return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateFailed, - ErrorMsg: dealError.String, - }, + HTTPCode: http.StatusNotFound, + } + } + + ret := &DealStatus{ + HTTPCode: http.StatusOK, + } + + if pdp_complete.Valid { + if pdp_complete.Bool { + ret.Response.DDOV1.State = DealStateComplete } } + if ddo_complete.Valid { + if ddo_complete.Bool { + ret.Response.DDOV1.State = DealStateComplete + } + } + + if ret.Response.DDOV1.State == DealStateComplete && ret.Response.PDPV1.State == DealStateComplete { + return ret + } + var waitingForPipeline bool - err = m.db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_pipeline_waiting WHERE id = $1)`, id.String()).Scan(&waitingForPipeline) + err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_pipeline_waiting WHERE id = $1)`, id.String()).Scan(&waitingForPipeline) if err != nil { log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) return &DealStatus{ @@ -49,12 +79,7 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { } } if waitingForPipeline { - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateAccepted, - }, - } + ret.Response.DDOV1.State = DealStateAccepted } var pdeals []struct { @@ -63,7 +88,7 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { Indexed bool `db:"indexed"` } - err = m.db.Select(ctx, &pdeals, `SELECT + err = m.DB.Select(ctx, &pdeals, `SELECT sector, sealed, indexed @@ -80,57 +105,47 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { } if len(pdeals) > 1 { - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateProcessing, - }, - } + ret.Response.DDOV1.State = DealStateProcessing } // If deal is still in pipeline if len(pdeals) == 1 { pdeal := pdeals[0] if pdeal.Sector == nil { - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateProcessing, - }, - } + ret.Response.DDOV1.State = DealStateProcessing } if !pdeal.Sealed { - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateSealing, - }, - } + ret.Response.DDOV1.State = DealStateSealing } if !pdeal.Indexed { - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateIndexing, - }, - } + ret.Response.DDOV1.State = DealStateIndexing } } - return &DealStatus{ - HTTPCode: http.StatusOK, - Response: &DealStatusResponse{ - State: DealStateComplete, - }, + var pdpPipeline bool + err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pdp_pipeline WHERE id = $1)`, id.String()).Scan(&pdpPipeline) + if err != nil { + log.Errorw("failed to query the db for PDP deal status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } } + if waitingForPipeline { + ret.Response.PDPV1.State = DealStateProcessing + } else { + ret.Response.PDPV1.State = DealStateAccepted + } + + return ret } +// Supported retrieves and returns maps of product names and data source names with their enabled status, or an error if the query fails. func (m *MK20) Supported(ctx context.Context) (map[string]bool, map[string]bool, error) { var products []struct { Name string `db:"name"` Enabled bool `db:"enabled"` } - err := m.db.Select(ctx, &products, `SELECT name, enabled FROM market_mk20_products`) + err := m.DB.Select(ctx, &products, `SELECT name, enabled FROM market_mk20_products`) if err != nil { return nil, nil, err } @@ -145,7 +160,7 @@ func (m *MK20) Supported(ctx context.Context) (map[string]bool, map[string]bool, Name string `db:"name"` Enabled bool `db:"enabled"` } - err = m.db.Select(ctx, &sources, `SELECT name, enabled FROM market_mk20_data_source`) + err = m.DB.Select(ctx, &sources, `SELECT name, enabled FROM market_mk20_data_source`) if err != nil { return nil, nil, err } diff --git a/market/mk20/mk20gen/gen.go b/market/mk20/mk20gen/gen.go index 374b8808e..b4a11385a 100644 --- a/market/mk20/mk20gen/gen.go +++ b/market/mk20/mk20gen/gen.go @@ -1,379 +1,965 @@ package main -import ( - "bytes" - "flag" - "fmt" - "go/ast" - "go/doc" - "go/token" - "go/types" - "log" - "os" - "sort" - "strings" - - "golang.org/x/tools/go/packages" -) - -type StructInfo struct { - Name string - Doc string - Fields []FieldInfo -} - -type FieldInfo struct { - Name string - Type string - Tag string - Doc string - Typ types.Type // ā† add this field -} - -type constEntry struct { - Name string - Value string - Doc string -} - -var visited = map[string]bool{} -var structMap = map[string]StructInfo{} -var rendered = map[string]bool{} -var constMap = map[string][]constEntry{} - -var skipTypes = map[string]bool{ - "ProviderDealRejectionInfo": true, - "DBDeal": true, - "dbProduct": true, - "dbDataSource": true, - "productAndDataSource": true, - "MK20": true, - "DealStatus": true, -} - -var includeConsts = map[string]bool{ - "ErrorCode": true, - "DealState": true, -} - -func main() { - var pkgPath, output string - flag.StringVar(&pkgPath, "pkg", "./", "Package to scan") - flag.StringVar(&output, "output", "info.md", "Output file") - flag.Parse() - - cfg := &packages.Config{ - Mode: packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports | packages.NeedName | packages.NeedFiles, - Fset: token.NewFileSet(), - } - - pkgs, err := packages.Load(cfg, pkgPath) - if err != nil { - log.Fatalf("Failed to load package: %v", err) - } - - for _, pkg := range pkgs { - docPkg, err := doc.NewFromFiles(cfg.Fset, pkg.Syntax, pkg.PkgPath) - if err != nil { - log.Fatalf("Failed to parse package: %v", err) - } - for _, t := range docPkg.Types { - if st, ok := t.Decl.Specs[0].(*ast.TypeSpec); ok { - if structType, ok := st.Type.(*ast.StructType); ok { - name := st.Name.Name - if visited[name] || skipTypes[name] { - continue - } - visited[name] = true - collectStruct(pkg, name, structType, t.Doc) - } - } - } - for _, file := range pkg.Syntax { - for _, decl := range file.Decls { - genDecl, ok := decl.(*ast.GenDecl) - if !ok || genDecl.Tok != token.CONST { - continue - } - - for _, spec := range genDecl.Specs { - vspec := spec.(*ast.ValueSpec) - for _, name := range vspec.Names { - obj := pkg.TypesInfo.Defs[name] - if obj == nil { - continue - } - typ := obj.Type().String() // e.g., "main.ErrCode" - parts := strings.Split(typ, ".") - typeName := parts[len(parts)-1] // just "ErrCode" - - if !includeConsts[typeName] { - continue - } - - if !rendered[typeName] { - constMap[typeName] = []constEntry{} - rendered[typeName] = true - } - - val := "" - if con, ok := obj.(*types.Const); ok { - val = con.Val().ExactString() - } - cdoc := strings.TrimSpace(vspec.Doc.Text()) - constMap[typeName] = append(constMap[typeName], constEntry{ - Name: name.Name, - Value: val, - Doc: cdoc, - }) - } - } - } - } - } - - writeOutput(output) -} - -func collectStruct(pkg *packages.Package, name string, structType *ast.StructType, docText string) { - info := StructInfo{ - Name: name, - Doc: strings.TrimSpace(docText), - } - - for _, field := range structType.Fields.List { - var fieldName string - if len(field.Names) > 0 { - fieldName = field.Names[0].Name - } else { - fieldName = fmt.Sprintf("%s", field.Type) - } - - var fieldType string - //if typ := pkg.TypesInfo.TypeOf(field.Type); typ != nil { - // fieldType = types.TypeString(typ, func(p *types.Package) string { - // return p.Name() - // }) - //} else { - // fieldType = fmt.Sprintf("%s", field.Type) - //} - var typ types.Type - if t := pkg.TypesInfo.TypeOf(field.Type); t != nil { - typ = t - fieldType = types.TypeString(t, func(p *types.Package) string { - return p.Name() - }) - } - - fieldTag := "" - if field.Tag != nil { - fieldTag = field.Tag.Value - } - - var fieldDoc string - if field.Doc != nil { - lines := strings.Split(field.Doc.Text(), "\n") - for i := range lines { - lines[i] = strings.TrimSpace(lines[i]) - } - fieldDoc = strings.Join(lines, " ") - } - - info.Fields = append(info.Fields, FieldInfo{ - Name: fieldName, - Type: fieldType, - Tag: fieldTag, - Doc: fieldDoc, - Typ: typ, - }) - - baseType := fieldType - - baseType = strings.TrimPrefix(baseType, "*") - - baseType = strings.TrimPrefix(baseType, "[]") - baseType = strings.Split(baseType, ".")[0] - if skipTypes[baseType] { - continue - } - if !visited[baseType] { - visited[baseType] = true - collectFromImports(baseType) - } - } - - structMap[name] = info -} - -func collectFromImports(typeName string) { - // future: support nested imports with doc.New(...) -} - -func writeOutput(path string) { - var buf bytes.Buffer - - buf.WriteString("# Storage Market Interface\n\n") - buf.WriteString("This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider.\n\n") - - buf.WriteString("## \U0001F4E1 MK20 HTTP API Overview\n\n") - buf.WriteString("The MK20 storage market module provides a set of HTTP endpoints under `/market/mk20` that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior.\n\n") - - buf.WriteString("### Base URL\n\n" + - "The base URL for all MK20 endpoints is: \n\n" + - "```\n\n/market/mk20\n\n```" + - "\n\n") - - buf.WriteString("### šŸ”„ POST /store\n\n") - buf.WriteString("Submit a new MK20 deal.\n\n") - buf.WriteString("- **Content-Type**: N/A\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**: N/A\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: Deal accepted\n") - buf.WriteString(" - Other [HTTP codes](#constants-for-errorcode) indicate validation failure, rejection, or system errors\n\n") - - buf.WriteString("### šŸ§¾ GET /status?id=\n\n") - buf.WriteString("Retrieve the current status of a deal.\n\n") - buf.WriteString("- **Content-Type**: `application/json`\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**:\n") - buf.WriteString(" - `id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: JSON-encoded [deal status](#dealstatusresponse) information\n") - buf.WriteString(" - `400 Bad Request`: Missing or invalid ID\n") - buf.WriteString(" - `500 Internal Server Error`: If backend fails to respond\n\n") - - buf.WriteString("### šŸ“œ GET /contracts\n\n") - buf.WriteString("- **Content-Type**: N/A\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**: N/A\n") - buf.WriteString("Return the list of contract addresses supported by the provider.\n\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: [JSON array of contract addresses](#supportedcontracts)\n") - buf.WriteString(" - `500 Internal Server Error`: Query or serialization failure\n\n") - - buf.WriteString("### šŸ—‚ PUT /data?id=\n\n") - buf.WriteString("Upload deal data after the deal has been accepted.\n\n") - buf.WriteString("- **Content-Type**: `application/octet-stream`\n") - buf.WriteString("- **Body**: Deal data bytes\n") - buf.WriteString("- **Query Parameter**:\n -`id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") - buf.WriteString("- **Headers**:\n") - buf.WriteString(" - `Content-Length`: must be deal's raw size\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: if data is successfully streamed\n") - buf.WriteString(" - `400`, `413`, or `415`: on validation failures\n\n") - - buf.WriteString("### šŸ§  GET /info\n\n") - buf.WriteString("- **Content-Type**: N/A\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**: N/A\n") - buf.WriteString("Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources.\n\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: with markdown content of the info file\n") - buf.WriteString(" - `500 Internal Server Error`: if file is not found or cannot be read\n\n") - - buf.WriteString("### šŸ§° GET /products\n\n") - buf.WriteString("- **Content-Type**: N/A\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**: N/A\n") - buf.WriteString("Fetch json list of the supported products.\n\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: with json content\n") - buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") - - buf.WriteString("### šŸŒ GET /sources\n\n") - buf.WriteString("- **Content-Type**: N/A\n") - buf.WriteString("- **Body**: N/A\n") - buf.WriteString("- **Query Parameters**: N/A\n") - buf.WriteString("Fetch json list of the supported data sources.\n\n") - buf.WriteString("- **Response**:\n") - buf.WriteString(" - `200 OK`: with json content\n") - buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") - - buf.WriteString("## Supported Deal Types\n\n") - buf.WriteString("This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.\n\n") - - ordered := []string{"Deal", "DataSource", "Products"} - var rest []string - for k := range structMap { - if k != "Deal" && k != "DataSource" && k != "Products" { - rest = append(rest, k) - } - } - sort.Strings(rest) - keys := append(ordered, rest...) - - for _, k := range keys { - s, ok := structMap[k] - if !ok { - continue - } - buf.WriteString(fmt.Sprintf("### %s\n\n", s.Name)) - if s.Doc != "" { - buf.WriteString(s.Doc + "\n\n") - } - buf.WriteString("| Field | Type | Tag | Description |\n") - buf.WriteString("|-------|------|-----|-------------|\n") - for _, f := range s.Fields { - typeName := f.Type - linkTarget := "" - - // Strip common wrappers like pointer/star and slice - trimmed := strings.TrimPrefix(typeName, "*") - trimmed = strings.TrimPrefix(trimmed, "[]") - parts := strings.Split(trimmed, ".") - baseType := parts[len(parts)-1] - - if _, ok := structMap[baseType]; ok { - linkTarget = fmt.Sprintf("[%s](#%s)", f.Type, strings.ToLower(baseType)) - } else if _, ok := constMap[baseType]; ok { - linkTarget = fmt.Sprintf("[%s](#constants-for-%s)", f.Type, strings.ToLower(baseType)) - } else { - typ := f.Typ - if ptr, ok := typ.(*types.Pointer); ok { - typ = ptr.Elem() - } - if named, ok := typ.(*types.Named); ok && named.Obj() != nil && named.Obj().Pkg() != nil { - - pkgPath := named.Obj().Pkg().Path() - objName := named.Obj().Name() - linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/%s#%s)", typeName, pkgPath, objName) - } else if typ != nil && typ.String() == baseType { - linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, baseType) - } else if slice, ok := typ.(*types.Slice); ok { - elem := slice.Elem() - if basic, ok := elem.(*types.Basic); ok { - linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, basic.Name()) - } else { - linkTarget = typeName - } - } else { - linkTarget = typeName - } - } - - buf.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", - f.Name, linkTarget, strings.Trim(f.Tag, "`"), f.Doc)) - } - buf.WriteString("\n") - } - - // Render constants with sort order - for k, v := range constMap { - if len(v) == 0 { - continue - } - buf.WriteString(fmt.Sprintf("### Constants for %s\n\n", k)) - buf.WriteString("| Constant | Code | Description |\n") - buf.WriteString("|----------|------|-------------|\n") - for _, c := range v { - buf.WriteString(fmt.Sprintf("| %s | %s | %s |\n", c.Name, c.Value, c.Doc)) - } - buf.WriteString("\n") - } - - err := os.WriteFile(path, buf.Bytes(), 0644) - if err != nil { - log.Fatalf("Failed to write output: %v", err) - } -} +// +//import ( +// "bytes" +// "flag" +// "fmt" +// "go/ast" +// "go/doc" +// "go/token" +// "go/types" +// "log" +// "os" +// "sort" +// "strings" +// +// "golang.org/x/tools/go/packages" +//) +// +//// Note: This file has too many static things. Go parse package is not easy to work with and +//// is a nightmare. Wasting month[s] to build a correct parses does not seem correct use of time. +// +//type StructInfo struct { +// Name string +// Doc string +// Fields []*FieldInfo +//} +// +//type FieldInfo struct { +// Name string +// Type string +// Tag string +// Doc string +// Typ types.Type +//} +// +//type constEntry struct { +// Name string +// Value string +// Doc string +//} +// +//var visited = map[string]bool{} +//var structMap = map[string]*StructInfo{} +//var rendered = map[string]bool{} +//var constMap = map[string][]constEntry{} +// +//var skipTypes = map[string]bool{ +// "ProviderDealRejectionInfo": true, +// "DBDeal": true, +// "dbProduct": true, +// "dbDataSource": true, +// "productAndDataSource": true, +// "MK20": true, +// "DealStatus": true, +//} +// +//var includeConsts = map[string]bool{ +// "DealCode": true, +// "DealState": true, +// "UploadStatusCode": true, +// "UploadStartCode": true, +// "UploadCode": true, +//} +// +////type ParamDoc struct { +//// Name string +//// Type string +//// Optional bool +//// Comment string +////} +////type ReturnDoc struct { +//// Name string +//// Type string +//// Comment string +////} +// +////// FunctionDoc holds extracted param and return comments for a function. +////type FunctionDoc struct { +//// Params []ParamDoc +//// Returns []ReturnDoc +////} +//// +////type handlerInfo struct { +//// Path string +//// Method string +//// FuncName string +//// Calls map[string]bool +//// Types map[string]bool +//// Constants map[string]bool +//// Errors map[string]bool +//// RequestBodyType string +//// ResponseBodyType string +////} +//// +////var allHandlers = map[string]*handlerInfo{} // key = function name +//// +////var httpCodes = map[string]struct { +//// Code string +//// Msg string +////}{ +//// "http.StatusBadRequest": { +//// Code: "400", +//// Msg: "Bad Request - Invalid input or validation error", +//// }, +//// "http.StatusOK": { +//// Code: "200", +//// Msg: "OK - Success", +//// }, +//// "http.StatusInternalServerError": { +//// Code: "500", +//// Msg: "Internal Server Error", +//// }, +////} +//// +////var ( +//// paramRe = regexp.MustCompile(`@param\s+(\w+)\s+([^\s\[]+)(\s+\[optional\])?(.*)`) +//// returnRe = regexp.MustCompile(`@Return\s+(\w+)?\s*([^\s\[]+)?(.*)`) +////) +// +////func ParseFunctionDocsFromComments(pkgPath string) map[string]*FunctionDoc { +//// fset := token.NewFileSet() +//// pkgs, err := parser.ParseDir(fset, pkgPath, nil, parser.ParseComments) +//// if err != nil { +//// panic(err) +//// } +//// +//// funcDocs := map[string]*FunctionDoc{} +//// +//// for _, pkg := range pkgs { +//// for _, file := range pkg.Files { +//// for _, decl := range file.Decls { +//// fn, ok := decl.(*ast.FuncDecl) +//// if !ok || fn.Doc == nil { +//// continue +//// } +//// +//// doc := &FunctionDoc{} +//// for _, c := range fn.Doc.List { +//// txt := strings.TrimSpace(strings.TrimPrefix(c.Text, "//")) +//// if m := paramRe.FindStringSubmatch(txt); m != nil { +//// doc.Params = append(doc.Params, ParamDoc{ +//// Name: m[1], +//// Type: m[2], +//// Optional: strings.Contains(m[3], "optional"), +//// Comment: strings.TrimSpace(m[4]), +//// }) +//// } else if m := returnRe.FindStringSubmatch(txt); m != nil { +//// doc.Returns = append(doc.Returns, ReturnDoc{ +//// Name: m[1], +//// Type: m[2], +//// Comment: strings.TrimSpace(m[3]), +//// }) +//// } +//// } +//// +//// if len(doc.Params) > 0 || len(doc.Returns) > 0 { +//// funcDocs[fn.Name.Name] = doc +//// } +//// } +//// } +//// } +//// return funcDocs +////} +// +//func main() { +// var pkgPath, output string +// flag.StringVar(&pkgPath, "pkg", "./", "Package to scan") +// flag.StringVar(&output, "output", "info.md", "Output file") +// flag.Parse() +// +// //pkgPath := "/Users/lexluthr/github/filecoin-project/curio/market/mk20" +// //routerFile := filepath.Join(pkgPath, "http", "http.go") +// +// cfg := &packages.Config{ +// Mode: packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedDeps | packages.NeedImports | packages.NeedName | packages.NeedFiles | packages.LoadAllSyntax, +// Fset: token.NewFileSet(), +// } +// +// pkgs, err := packages.Load(cfg, pkgPath) +// if err != nil { +// log.Fatalf("Failed to load package: %v", err) +// } +// +// for _, pkg := range pkgs { +// docPkg, err := doc.NewFromFiles(cfg.Fset, pkg.Syntax, pkg.PkgPath) +// if err != nil { +// log.Fatalf("Failed to parse package: %v", err) +// } +// for _, t := range docPkg.Types { +// if st, ok := t.Decl.Specs[0].(*ast.TypeSpec); ok { +// if structType, ok := st.Type.(*ast.StructType); ok { +// name := st.Name.Name +// if visited[name] || skipTypes[name] { +// continue +// } +// visited[name] = true +// collectStruct(pkg, name, structType, t.Doc) +// } +// } +// } +// for _, file := range pkg.Syntax { +// for _, decl := range file.Decls { +// genDecl, ok := decl.(*ast.GenDecl) +// if !ok || genDecl.Tok != token.CONST { +// continue +// } +// +// for _, spec := range genDecl.Specs { +// vspec := spec.(*ast.ValueSpec) +// for _, name := range vspec.Names { +// obj := pkg.TypesInfo.Defs[name] +// if obj == nil { +// continue +// } +// typ := obj.Type().String() // e.g., "main.ErrCode" +// parts := strings.Split(typ, ".") +// typeName := parts[len(parts)-1] // just "ErrCode" +// +// if !includeConsts[typeName] { +// continue +// } +// +// if !rendered[typeName] { +// constMap[typeName] = []constEntry{} +// rendered[typeName] = true +// } +// +// val := "" +// if con, ok := obj.(*types.Const); ok { +// val = con.Val().ExactString() +// } +// cdoc := strings.TrimSpace(vspec.Doc.Text()) +// constMap[typeName] = append(constMap[typeName], constEntry{ +// Name: name.Name, +// Value: val, +// Doc: cdoc, +// }) +// } +// } +// } +// } +// } +// +// //fm := ParseFunctionDocsFromComments(pkgPath) +// //for fname, doc := range fm { +// // fmt.Printf("Function: %s\n", fname) +// // if len(doc.Params) > 0 { +// // fmt.Println(" Params:") +// // for _, p := range doc.Params { +// // fmt.Printf(" - %s %s", p.Name, p.Type) +// // if p.Optional { +// // fmt.Print(" (optional)") +// // } +// // if p.Comment != "" { +// // fmt.Printf(" -- %s", p.Comment) +// // } +// // fmt.Println() +// // } +// // } +// // if len(doc.Returns) > 0 { +// // fmt.Println(" Returns:") +// // for _, r := range doc.Returns { +// // fmt.Printf(" - Name: %s Type: %s", r.Name, r.Type) +// // if r.Comment != "" { +// // fmt.Printf(" -- Comment: %s", r.Comment) +// // } +// // fmt.Println() +// // } +// // } +// //} +// +// writeOutput(output) +// //parseMux(routerFile) +// //fmt.Println("Done tracing handlers") +// //parseHandlerBodies(routerFile) +// //fmt.Println("Done parsing handler bodies") +// //for k, v := range allHandlers { +// // fmt.Println("------------------") +// // fmt.Println("Name:", k) +// // fmt.Println("Path:", v.Path) +// // fmt.Println("Method:", v.Method) +// // fmt.Println("Constants", v.Constants) +// // fmt.Println("Calls:", v.Calls) +// // fmt.Println("Types:", v.Types) +// // fmt.Println("RequestBody", v.RequestBodyType) +// // fmt.Println("ResponseBody", v.ResponseBodyType) +// // fmt.Println("------------------") +// //} +// //fmt.Println("----------------") +// //fmt.Println("----------------") +// //for k, v := range constMap { +// // fmt.Println("Name:", k) +// // for _, e := range v { +// // fmt.Printf(" - %s: %s\n", e.Name, e.Value) +// // } +// //} +// //fmt.Println("----------------") +// //fmt.Println("----------------") +// //for _, h := range allHandlers { +// // fmt.Printf("%s %s\n", h.Method, h.Path) +// // // Optional: print summary from docs if available +// // // Parameters +// // mainCall := "" +// // for call := range h.Calls { +// // if strings.HasPrefix(call, "mk20.") { +// // mainCall = strings.TrimPrefix(call, "mk20.") +// // break +// // } +// // } +// // if mainCall != "" { +// // fmt.Println("### Parameters") +// // if doc, ok := fm[mainCall]; ok { +// // for _, param := range doc.Params { +// // fmt.Printf("- %s (%s)%s\n", param.Name, param.Type, +// // func() string { +// // if param.Optional { +// // return " [optional]" +// // } +// // return "" +// // }()) +// // } +// // } else if len(h.Types) > 0 { +// // // fallback: print type +// // for typ := range h.Types { +// // fmt.Printf("- body (%s)\n", typ) +// // } +// // } +// // } +// // // Responses +// // fmt.Println("### Possible Responses") +// // for code := range h.Constants { +// // switch code { +// // case "http.StatusBadRequest": +// // fmt.Println("- 400 Bad Request: Invalid input or validation error.") +// // case "http.StatusOK": +// // fmt.Println("- 200 OK: Success.") +// // case "http.StatusInternalServerError": +// // fmt.Println("- 500 Internal Server Error.") +// // // ... add more as needed +// // default: +// // fmt.Printf("- %s\n", code) +// // } +// // } +// // fmt.Println() +// //} +// +// //formatHandlerDocs(fm) +// //generateSwaggoComments(fm) +// +//} +// +////func extractPathParams(path string) []string { +//// var out []string +//// for _, part := range strings.Split(path, "/") { +//// if strings.HasPrefix(part, "{") && strings.HasSuffix(part, "}") { +//// out = append(out, strings.TrimSuffix(strings.TrimPrefix(part, "{"), "}")) +//// } +//// } +//// return out +////} +// +////func generateSwaggoComments(funcDocs map[string]*FunctionDoc) { +//// for _, h := range allHandlers { +//// fmt.Printf("// @Router %s [%s]\n", h.Path, strings.ToLower(h.Method)) +//// +//// // Path parameters from {id}, {chunkNum}, etc. +//// for _, param := range extractPathParams(h.Path) { +//// fmt.Printf("// @Param %s path string true \"%s\"\n", param, param) +//// } +//// +//// // Request body +//// if h.RequestBodyType != "" { +//// fmt.Println("// @accepts json") +//// fmt.Printf("// @Param body body %s true\n", h.RequestBodyType) +//// fmt.Println("// @Accept json\n// @Produce json") +//// } else if h.Method == "PUT" { +//// fmt.Println("// @accepts bytes") +//// fmt.Printf("// @Param body body []byte true \"raw binary\"\n") +//// } +//// +//// // Figure out function called like mk20.Something +//// var mk20Call string +//// for call := range h.Calls { +//// if strings.HasPrefix(call, "mk20.") { +//// mk20Call = strings.TrimPrefix(call, "mk20.") +//// break +//// } +//// } +//// +//// // Return codes (Swagger `@Success` / `@Failure`) +//// hasReturn := false +//// if doc, ok := funcDocs[mk20Call]; ok { +//// for _, ret := range doc.Returns { +//// key := strings.TrimPrefix(ret.Name, "*") +//// key = strings.TrimPrefix(key, "mk20.") +//// if entries, ok := constMap[key]; ok { +//// for _, entry := range entries { +//// msg := strings.TrimSuffix(entry.Doc, ".") +//// tag := "@Failure" +//// if strings.HasPrefix(fmt.Sprintf("%d", entry.Value), "2") { +//// tag = "@Success" +//// hasReturn = true +//// } else { +//// fmt.Printf("// %s %s {object} %s \"%s\"\n", tag, entry.Value, key, msg) +//// } +//// } +//// } +//// } +//// // Fallback to direct http constants if nothing above +//// for k := range h.Constants { +//// if msg, ok := httpCodes[k]; ok { +//// tag := "@Failure" +//// if strings.HasPrefix(fmt.Sprintf("%d", msg.Code), "2") { +//// tag = "@Success" +//// hasReturn = true +//// } else { +//// fmt.Printf("// %s %s {string} string \"%s\"\n", tag, msg.Code, msg.Msg) +//// } +//// +//// } +//// } +//// +//// // If known response type +//// if h.ResponseBodyType != "" && hasReturn { +//// fmt.Println("// @produce json") +//// fmt.Printf("// @Success 200 {object} %s\n", h.ResponseBodyType) +//// } +//// } else { +//// // Fallback to direct http constants if nothing above +//// for k := range h.Constants { +//// if msg, ok := httpCodes[k]; ok { +//// tag := "@Failure" +//// if strings.HasPrefix(fmt.Sprintf("%d", msg.Code), "2") { +//// tag = "@Success" +//// hasReturn = true +//// } else { +//// fmt.Printf("// %s %s {string} string \"%s\"\n", tag, msg.Code, msg.Msg) +//// } +//// //fmt.Printf("// %s %s {string} string \"%s\"\n", tag, msg.Code, msg.Msg) +//// } +//// } +//// +//// // If known response type +//// if h.ResponseBodyType != "" && hasReturn { +//// fmt.Println("// @produce json") +//// fmt.Printf("// @Success 200 {object} %s\n", h.ResponseBodyType) +//// } +//// } +//// +//// fmt.Println() +//// } +////} +//// +////func formatHandlerDocs(funcDocs map[string]*FunctionDoc) { +//// for _, h := range allHandlers { +//// fmt.Printf("%s %s\n", h.Method, h.Path) +//// +//// // 1. Find the mk20 call +//// var mk20Call string +//// for call := range h.Calls { +//// if strings.HasPrefix(call, "mk20.") { +//// mk20Call = strings.TrimPrefix(call, "mk20.") +//// //fmt.Println("mk20Call: ", mk20Call) +//// break +//// } +//// } +//// +//// // 2. Look up params and returns +//// doc, ok := funcDocs[mk20Call] +//// if ok { +//// if h.RequestBodyType != "" { +//// fmt.Printf("### Request Body\n- %s\n", h.RequestBodyType) +//// } +//// if h.RequestBodyType == "" && h.Method == "PUT" { +//// fmt.Printf("### Request Body\n- bytes\n") +//// } +//// +//// // 3. Lookup constMap based on return types +//// fmt.Println("### Possible Responses") +//// for _, ret := range doc.Returns { +//// key := strings.TrimPrefix(ret.Name, "*") +//// key = strings.TrimPrefix(key, "mk20.") +//// if entries, ok := constMap[key]; ok { +//// for _, entry := range entries { +//// comment := entry.Doc +//// comment = strings.TrimSuffix(comment, ".") +//// if comment == "" { +//// fmt.Printf("- %s: %s\n", entry.Value, entry.Name) +//// } else { +//// fmt.Printf("- %s: %s - %s\n", entry.Value, entry.Name, comment) +//// } +//// } +//// } +//// } +//// for k, _ := range h.Constants { +//// fmt.Printf("- %s\n", httpCodes[k]) +//// } +//// if h.ResponseBodyType != "" { +//// fmt.Printf("### Response Body\n- %s\n", h.ResponseBodyType) +//// } +//// } else { +//// //fmt.Println("### Parameters") +//// if h.RequestBodyType != "" { +//// fmt.Printf("### Request Body\n- %s\n", h.RequestBodyType) +//// } +//// if h.RequestBodyType == "" && h.Method == "PUT" { +//// fmt.Printf("### Request Body\n- bytes\n") +//// } +//// fmt.Println("### Possible Responses") +//// for k, _ := range h.Constants { +//// fmt.Printf("- %s\n", httpCodes[k]) +//// } +//// if h.ResponseBodyType != "" { +//// fmt.Printf("### Response Body\n- %s\n", h.ResponseBodyType) +//// } +//// } +//// fmt.Println() +//// } +////} +// +//func collectStruct(pkg *packages.Package, name string, structType *ast.StructType, docText string) { +// info := StructInfo{ +// Name: name, +// Doc: strings.TrimSpace(docText), +// } +// +// for _, field := range structType.Fields.List { +// var fieldName string +// if len(field.Names) > 0 { +// fieldName = field.Names[0].Name +// } else { +// fieldName = fmt.Sprintf("%s", field.Type) +// } +// +// var fieldType string +// var typ types.Type +// if t := pkg.TypesInfo.TypeOf(field.Type); t != nil { +// typ = t +// fieldType = types.TypeString(t, func(p *types.Package) string { +// return p.Name() +// }) +// } +// +// fieldTag := "" +// if field.Tag != nil { +// fieldTag = field.Tag.Value +// } +// +// var fieldDoc string +// if field.Doc != nil { +// lines := strings.Split(field.Doc.Text(), "\n") +// for i := range lines { +// lines[i] = strings.TrimSpace(lines[i]) +// } +// fieldDoc = strings.Join(lines, " ") +// } +// +// info.Fields = append(info.Fields, &FieldInfo{ +// Name: fieldName, +// Type: fieldType, +// Tag: fieldTag, +// Doc: fieldDoc, +// Typ: typ, +// }) +// +// baseType := fieldType +// +// baseType = strings.TrimPrefix(baseType, "*") +// +// baseType = strings.TrimPrefix(baseType, "[]") +// baseType = strings.Split(baseType, ".")[0] +// if skipTypes[baseType] { +// continue +// } +// if !visited[baseType] { +// visited[baseType] = true +// collectFromImports(baseType) +// } +// } +// +// structMap[name] = &info +//} +// +//func collectFromImports(typeName string) { +// // future: support nested imports with doc.New(...) +//} +// +//func writeOutput(path string) { +// var buf bytes.Buffer +// +// buf.WriteString("# Storage Market Interface\n\n") +// buf.WriteString("This document describes the storage market types and supported HTTP methods for making deals with Curio Storage Provider.\n\n") +// +// buf.WriteString("## \U0001F4E1 MK20 HTTP API Overview\n\n") +// buf.WriteString("The MK20 storage market module provides a set of HTTP endpoints under `/market/mk20` that allow clients to submit, track, and finalize storage deals with storage providers. This section documents all available routes and their expected behavior.\n\n") +// +// buf.WriteString("### Base URL\n\n" + +// "The base URL for all MK20 endpoints is: \n\n" + +// "```\n\n/market/mk20\n\n```" + +// "\n\n") +// +// buf.WriteString("### šŸ”„ POST /store\n\n") +// buf.WriteString("Submit a new MK20 deal.\n\n") +// buf.WriteString("- **Content-Type**: N/A\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**: N/A\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: Deal accepted\n") +// buf.WriteString(" - Other [HTTP codes](#constants-for-errorcode) indicate validation failure, rejection, or system errors\n\n") +// +// buf.WriteString("### šŸ§¾ GET /status?id=\n\n") +// buf.WriteString("Retrieve the current status of a deal.\n\n") +// buf.WriteString("- **Content-Type**: `application/json`\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**:\n") +// buf.WriteString(" - `id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: JSON-encoded [deal status](#dealstatusresponse) information\n") +// buf.WriteString(" - `400 Bad Request`: Missing or invalid ID\n") +// buf.WriteString(" - `500 Internal Server Error`: If backend fails to respond\n\n") +// +// buf.WriteString("### šŸ“œ GET /contracts\n\n") +// buf.WriteString("- **Content-Type**: N/A\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**: N/A\n") +// buf.WriteString("Return the list of contract addresses supported by the provider.\n\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: [JSON array of contract addresses](#supportedcontracts)\n") +// buf.WriteString(" - `500 Internal Server Error`: Query or serialization failure\n\n") +// +// buf.WriteString("### šŸ—‚ PUT /data?id=\n\n") +// buf.WriteString("Upload deal data after the deal has been accepted.\n\n") +// buf.WriteString("- **Content-Type**: `application/octet-stream`\n") +// buf.WriteString("- **Body**: Deal data bytes\n") +// buf.WriteString("- **Query Parameter**:\n -`id`: Deal identifier in [ULID](https://github.com/ulid/spec) format\n") +// buf.WriteString("- **Headers**:\n") +// buf.WriteString(" - `Content-Length`: must be deal's raw size\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: if data is successfully streamed\n") +// buf.WriteString(" - `400`, `413`, or `415`: on validation failures\n\n") +// +// buf.WriteString("### šŸ§  GET /info\n\n") +// buf.WriteString("- **Content-Type**: N/A\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**: N/A\n") +// buf.WriteString("Fetch markdown-formatted documentation that describes the supported deal schema, products, and data sources.\n\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: with markdown content of the info file\n") +// buf.WriteString(" - `500 Internal Server Error`: if file is not found or cannot be read\n\n") +// +// buf.WriteString("### šŸ§° GET /products\n\n") +// buf.WriteString("- **Content-Type**: N/A\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**: N/A\n") +// buf.WriteString("Fetch json list of the supported products.\n\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: with json content\n") +// buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") +// +// buf.WriteString("### šŸŒ GET /sources\n\n") +// buf.WriteString("- **Content-Type**: N/A\n") +// buf.WriteString("- **Body**: N/A\n") +// buf.WriteString("- **Query Parameters**: N/A\n") +// buf.WriteString("Fetch json list of the supported data sources.\n\n") +// buf.WriteString("- **Response**:\n") +// buf.WriteString(" - `200 OK`: with json content\n") +// buf.WriteString(" - `500 Internal Server Error`: if info cannot be read\n\n") +// +// buf.WriteString("## Supported Deal Types\n\n") +// buf.WriteString("This document lists the data types and fields supported in the new storage market interface. It defines the deal structure, accepted data sources, and optional product extensions. Clients should use these definitions to format and validate their deals before submission.\n\n") +// +// ordered := []string{"Deal", "DataSource", "Products"} +// var rest []string +// for k := range structMap { +// if k != "Deal" && k != "DataSource" && k != "Products" { +// rest = append(rest, k) +// } +// } +// sort.Strings(rest) +// keys := append(ordered, rest...) +// +// for _, k := range keys { +// s, ok := structMap[k] +// if !ok { +// continue +// } +// buf.WriteString(fmt.Sprintf("### %s\n\n", s.Name)) +// if s.Doc != "" { +// buf.WriteString(s.Doc + "\n\n") +// } +// buf.WriteString("| Field | Type | Tag | Description |\n") +// buf.WriteString("|-------|------|-----|-------------|\n") +// for _, f := range s.Fields { +// typeName := f.Type +// linkTarget := "" +// +// // Strip common wrappers like pointer/star and slice +// trimmed := strings.TrimPrefix(typeName, "*") +// trimmed = strings.TrimPrefix(trimmed, "[]") +// parts := strings.Split(trimmed, ".") +// baseType := parts[len(parts)-1] +// +// if _, ok := structMap[baseType]; ok { +// linkTarget = fmt.Sprintf("[%s](#%s)", f.Type, strings.ToLower(baseType)) +// } else if _, ok := constMap[baseType]; ok { +// linkTarget = fmt.Sprintf("[%s](#constants-for-%s)", f.Type, strings.ToLower(baseType)) +// } else { +// typ := f.Typ +// if ptr, ok := typ.(*types.Pointer); ok { +// typ = ptr.Elem() +// } +// if named, ok := typ.(*types.Named); ok && named.Obj() != nil && named.Obj().Pkg() != nil { +// +// pkgPath := named.Obj().Pkg().Path() +// objName := named.Obj().Name() +// linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/%s#%s)", typeName, pkgPath, objName) +// } else if typ != nil && typ.String() == baseType { +// linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, baseType) +// } else if slice, ok := typ.(*types.Slice); ok { +// elem := slice.Elem() +// if basic, ok := elem.(*types.Basic); ok { +// linkTarget = fmt.Sprintf("[%s](https://pkg.go.dev/builtin#%s)", typeName, basic.Name()) +// } else { +// linkTarget = typeName +// } +// } else { +// linkTarget = typeName +// } +// } +// +// buf.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", +// f.Name, linkTarget, strings.Trim(f.Tag, "`"), f.Doc)) +// } +// buf.WriteString("\n") +// } +// +// // Render constants with sort order +// for k, v := range constMap { +// if len(v) == 0 { +// continue +// } +// buf.WriteString(fmt.Sprintf("### Constants for %s\n\n", k)) +// buf.WriteString("| Constant | Code | Description |\n") +// buf.WriteString("|----------|------|-------------|\n") +// for _, c := range v { +// buf.WriteString(fmt.Sprintf("| %s | %s | %s |\n", c.Name, c.Value, c.Doc)) +// } +// buf.WriteString("\n") +// } +// +// os.Stdout.WriteString(buf.String()) +// +// err := os.WriteFile(path, buf.Bytes(), 0644) +// if err != nil { +// log.Fatalf("Failed to write output: %v", err) +// } +//} +// +////func parseMux(path string) { +//// fset := token.NewFileSet() +//// +//// node, err := parser.ParseFile(fset, path, nil, 0) +//// if err != nil { +//// log.Fatalf("Parse error: %v", err) +//// } +//// +//// ast.Inspect(node, func(n ast.Node) bool { +//// call, ok := n.(*ast.CallExpr) +//// if !ok || len(call.Args) < 2 { +//// return true +//// } +//// +//// sel, ok := call.Fun.(*ast.SelectorExpr) +//// if !ok { +//// return true +//// } +//// +//// method := sel.Sel.Name +//// var path string +//// var fnName string +//// +//// switch method { +//// case "Get", "Post", "Put", "Delete": +//// if len(call.Args) != 2 { +//// return true +//// } +//// pathLit, ok := call.Args[0].(*ast.BasicLit) +//// if !ok || pathLit.Kind != token.STRING { +//// return true +//// } +//// method = strings.ToUpper(method) +//// path = strings.Trim(pathLit.Value, "\"") +//// fnName = call.Args[1].(*ast.SelectorExpr).Sel.Name +//// +//// case "Method": +//// if len(call.Args) != 3 { +//// return true +//// } +//// methodLit, ok := call.Args[0].(*ast.BasicLit) +//// if !ok || methodLit.Kind != token.STRING { +//// return true +//// } +//// method = strings.Trim(methodLit.Value, "\"") +//// +//// pathLit, ok := call.Args[1].(*ast.BasicLit) +//// if !ok || pathLit.Kind != token.STRING { +//// return true +//// } +//// path = strings.Trim(pathLit.Value, "\"") +//// fnName = extractHandlerFunc(call.Args[2]) +//// +//// default: +//// return true +//// } +//// +//// allHandlers[fnName] = &handlerInfo{ +//// Path: path, +//// Method: method, +//// FuncName: fnName, +//// Errors: map[string]bool{}, +//// } +//// return true +//// }) +////} +// +////func extractHandlerFunc(expr ast.Expr) string { +//// call, ok := expr.(*ast.CallExpr) +//// if !ok { +//// return "unknown" +//// } +//// +//// switch fun := call.Fun.(type) { +//// case *ast.SelectorExpr: +//// if fun.Sel.Name == "TimeoutHandler" && len(call.Args) > 0 { +//// return extractHandlerFunc(call.Args[0]) +//// } +//// if fun.Sel.Name == "HandlerFunc" && len(call.Args) > 0 { +//// if sel, ok := call.Args[0].(*ast.SelectorExpr); ok { +//// return sel.Sel.Name +//// } +//// } +//// } +//// return "unknown" +////} +// +////func parseHandlerBodies(path string) { +//// fset := token.NewFileSet() +//// file, err := parser.ParseFile(fset, path, nil, parser.AllErrors|parser.ParseComments) +//// if err != nil { +//// log.Fatalf("Parse error: %v", err) +//// } +//// for _, decl := range file.Decls { +//// fn, ok := decl.(*ast.FuncDecl) +//// if !ok || fn.Body == nil { +//// continue +//// } +//// name := fn.Name.Name +//// handler, exists := allHandlers[name] +//// if !exists { +//// continue +//// } +//// +//// calls := map[string]bool{} +//// types := map[string]bool{} +//// constants := map[string]bool{} +//// var reqType string +//// var respType string +//// +//// ast.Inspect(fn.Body, func(n ast.Node) bool { +//// switch node := n.(type) { +//// case *ast.CallExpr: +//// if sel, ok := node.Fun.(*ast.SelectorExpr); ok { +//// // http.WriteHeader or http.Error +//// if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "http" { +//// if sel.Sel.Name == "WriteHeader" || sel.Sel.Name == "Error" { +//// calls["http."+sel.Sel.Name] = true +//// } +//// } +//// // mdh.dm.MK20Handler. +//// if x1, ok := sel.X.(*ast.SelectorExpr); ok { +//// if x2, ok := x1.X.(*ast.SelectorExpr); ok { +//// if x2.X.(*ast.Ident).Name == "mdh" && +//// x2.Sel.Name == "dm" && +//// x1.Sel.Name == "MK20Handler" { +//// calls["mk20."+sel.Sel.Name] = true +//// } +//// } +//// } +//// // Detect json.Unmarshal(b, &type) +//// if sel.Sel.Name == "Unmarshal" && len(node.Args) == 2 { +//// if unary, ok := node.Args[1].(*ast.UnaryExpr); ok { +//// if ident, ok := unary.X.(*ast.Ident); ok { +//// reqType = findVarType(fn, ident.Name) +//// } +//// } +//// } +//// // Detect json.Marshal(type) +//// if sel.Sel.Name == "Marshal" && len(node.Args) == 1 { +//// if ident, ok := node.Args[0].(*ast.Ident); ok { +//// respType = findVarType(fn, ident.Name) +//// } +//// } +//// } +//// case *ast.AssignStmt: +//// for _, rhs := range node.Rhs { +//// if ce, ok := rhs.(*ast.CallExpr); ok { +//// if sel, ok := ce.Fun.(*ast.SelectorExpr); ok { +//// if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "http" { +//// if strings.HasPrefix(sel.Sel.Name, "Status") { +//// constants["http."+sel.Sel.Name] = true +//// } +//// } +//// } +//// } +//// } +//// case *ast.ValueSpec: +//// if node.Type != nil { +//// if se, ok := node.Type.(*ast.SelectorExpr); ok { +//// if ident, ok := se.X.(*ast.Ident); ok && ident.Name == "mk20" { +//// types["mk20."+se.Sel.Name] = true +//// } +//// } +//// } +//// case *ast.SelectorExpr: +//// if ident, ok := node.X.(*ast.Ident); ok && ident.Name == "http" { +//// if strings.HasPrefix(node.Sel.Name, "Status") { +//// constants["http."+node.Sel.Name] = true +//// } +//// } +//// } +//// return true +//// }) +//// +//// handler.Calls = calls +//// handler.Types = types +//// handler.Constants = constants +//// handler.RequestBodyType = reqType +//// handler.ResponseBodyType = respType +//// } +////} +//// +////// Helper to find type of variable declared in function scope +////func findVarType(fn *ast.FuncDecl, name string) string { +//// for _, stmt := range fn.Body.List { +//// if ds, ok := stmt.(*ast.DeclStmt); ok { +//// if gd, ok := ds.Decl.(*ast.GenDecl); ok { +//// for _, spec := range gd.Specs { +//// if vs, ok := spec.(*ast.ValueSpec); ok { +//// for _, ident := range vs.Names { +//// if ident.Name == name { +//// if se, ok := vs.Type.(*ast.SelectorExpr); ok { +//// if x, ok := se.X.(*ast.Ident); ok { +//// return x.Name + "." + se.Sel.Name +//// } +//// } +//// } +//// } +//// } +//// } +//// } +//// } +//// } +//// return "" +////} diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 8f3cf06e4..17e4a5a06 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -2,7 +2,6 @@ package mk20 import ( "context" - "net/http" "golang.org/x/xerrors" @@ -32,7 +31,7 @@ type PDPV1 struct { ExtraData []byte `json:"extra_data"` } -func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { +func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { code, err := IsProductEnabled(db, p.ProductName()) if err != nil { return code, err @@ -76,7 +75,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e var exists bool err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } if !exists { return ErrBadProposal, xerrors.Errorf("proofset does not exist") @@ -91,7 +90,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e var exists bool err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } if !exists { return ErrBadProposal, xerrors.Errorf("proofset does not exist") @@ -106,7 +105,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, e var exists bool err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) if err != nil { - return http.StatusInternalServerError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } if len(p.ExtraData) == 0 { return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") diff --git a/market/mk20/retrieval_v1.go b/market/mk20/retrieval_v1.go index 365771b58..500263780 100644 --- a/market/mk20/retrieval_v1.go +++ b/market/mk20/retrieval_v1.go @@ -19,7 +19,7 @@ type RetrievalV1 struct { AnnouncePiece bool `json:"announce_piece"` } -func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { +func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { code, err := IsProductEnabled(db, r.ProductName()) if err != nil { return code, err diff --git a/market/mk20/types.go b/market/mk20/types.go index 4f0e2149c..26c961bd2 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -21,9 +21,6 @@ type Deal struct { // Client wallet for the deal Client address.Address `json:"client"` - // Signature bytes for the client deal - Signature []byte `json:"signature"` - // Data represents the source of piece data and associated metadata. Data *DataSource `json:"data"` @@ -137,50 +134,60 @@ const ( // AggregateTypeNone represents the default aggregation type, indicating no specific aggregation is applied. AggregateTypeNone AggregateType = iota - // AggregateTypeV1 represents the first version of the aggregate type in the system. + // AggregateTypeV1 represents the first version of the aggregate type in the system. This is current PODSI aggregation + // based on https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0058.md AggregateTypeV1 ) -// ErrorCode represents an error code as an integer value -type ErrorCode int +// DealCode represents an error code as an integer value +type DealCode int const ( // Ok represents a successful operation with an HTTP status code of 200. - Ok ErrorCode = 200 + Ok DealCode = 200 + + // ErrUnAuthorized represents an error indicating unauthorized access with the code 401. + ErrUnAuthorized DealCode = 401 // ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic. - ErrBadProposal ErrorCode = 400 + ErrBadProposal DealCode = 400 + + // ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404. + ErrDealNotFound DealCode = 404 // ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data. - ErrMalformedDataSource ErrorCode = 430 + ErrMalformedDataSource DealCode = 430 // ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context. - ErrUnsupportedDataSource ErrorCode = 422 + ErrUnsupportedDataSource DealCode = 422 // ErrUnsupportedProduct indicates that the requested product is not supported by the provider. - ErrUnsupportedProduct ErrorCode = 423 + ErrUnsupportedProduct DealCode = 423 // ErrProductNotEnabled indicates that the requested product is not enabled on the provider. - ErrProductNotEnabled ErrorCode = 424 + ErrProductNotEnabled DealCode = 424 // ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data. - ErrProductValidationFailed ErrorCode = 425 + ErrProductValidationFailed DealCode = 425 // ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules. - ErrDealRejectedByMarket ErrorCode = 426 + ErrDealRejectedByMarket DealCode = 426 + + // ErrServerInternalError indicates an internal server error with a corresponding error code of 500. + ErrServerInternalError DealCode = 500 // ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503. - ErrServiceMaintenance ErrorCode = 503 + ErrServiceMaintenance DealCode = 503 // ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment. - ErrServiceOverloaded ErrorCode = 429 + ErrServiceOverloaded DealCode = 429 // ErrMarketNotEnabled indicates that the market is not enabled for the requested operation. - ErrMarketNotEnabled ErrorCode = 440 + ErrMarketNotEnabled DealCode = 440 // ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold. - ErrDurationTooShort ErrorCode = 441 + ErrDurationTooShort DealCode = 441 ) // ProductName represents a type for defining the product name identifier used in various operations and validations. @@ -205,6 +212,66 @@ const ( ) type product interface { - Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) + Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) ProductName() ProductName } + +// UploadStatusCode defines the return codes for the upload status +type UploadStatusCode int + +const ( + + // UploadStatusCodeOk represents a successful upload operation with status code 200. + UploadStatusCodeOk UploadStatusCode = 200 + + // UploadStatusCodeDealNotFound indicates that the requested deal was not found, corresponding to status code 404. + UploadStatusCodeDealNotFound UploadStatusCode = 404 + + // UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet. + UploadStatusCodeUploadNotStarted UploadStatusCode = 425 + + // UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500. + UploadStatusCodeServerError UploadStatusCode = 500 +) + +// UploadStartCode represents an integer type for return codes related to the upload start process. +type UploadStartCode int + +const ( + + // UploadStartCodeOk indicates a successful upload start request with status code 200. + UploadStartCodeOk UploadStartCode = 200 + + // UploadStartCodeBadRequest indicates a bad upload start request error with status code 400. + UploadStartCodeBadRequest UploadStartCode = 400 + + // UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process. + UploadStartCodeDealNotFound UploadStartCode = 404 + + // UploadStartCodeAlreadyStarted indicates that the upload process has already been initiated and cannot be started again. + UploadStartCodeAlreadyStarted UploadStartCode = 409 + + // UploadStartCodeServerError indicates an error occurred on the server while processing an upload start request. + UploadStartCodeServerError UploadStartCode = 500 +) + +// UploadCode represents return codes related to upload operations, typically based on HTTP status codes. +type UploadCode int + +const ( + + // UploadOk indicates a successful upload operation, represented by the HTTP status code 200. + UploadOk UploadCode = 200 + + // UploadBadRequest represents a bad request error with an HTTP status code of 400. + UploadBadRequest UploadCode = 400 + + // UploadNotFound represents an error where the requested upload chunk could not be found, typically corresponding to HTTP status 404. + UploadNotFound UploadCode = 404 + + // UploadChunkAlreadyUploaded indicates that the chunk has already been uploaded and cannot be re-uploaded. + UploadChunkAlreadyUploaded UploadCode = 409 + + // UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500. + UploadServerError UploadCode = 500 +) diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 325955fb5..b9d2c9e84 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -1,17 +1,23 @@ package mk20 import ( + "bytes" "context" + "crypto/ed25519" "crypto/rand" + "crypto/sha256" "database/sql" + "encoding/base64" "encoding/json" "errors" "fmt" "net/http" "net/url" + "strings" "time" "github.com/ipfs/go-cid" + "github.com/mr-tron/base58" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" @@ -19,7 +25,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/crypto" + fcrypto "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -28,17 +34,17 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" ) -func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { +func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { if d.Client.Empty() { return ErrBadProposal, xerrors.Errorf("no client") } - code, err := d.ValidateSignature() - if err != nil { - return code, xerrors.Errorf("signature validation failed: %w", err) - } + //code, err := d.ValidateSignature() + //if err != nil { + // return code, xerrors.Errorf("signature validation failed: %w", err) + //} - code, err = d.Products.Validate(db, cfg) + code, err := d.Products.Validate(db, cfg) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) } @@ -52,35 +58,35 @@ func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, er return Ok, nil } -func (d *Deal) ValidateSignature() (ErrorCode, error) { - if len(d.Signature) == 0 { - return ErrBadProposal, xerrors.Errorf("no signature") - } - - sig := &crypto.Signature{} - err := sig.UnmarshalBinary(d.Signature) - if err != nil { - return ErrBadProposal, xerrors.Errorf("invalid signature") - } - - msg, err := d.Identifier.MarshalBinary() - if err != nil { - return ErrBadProposal, xerrors.Errorf("invalid identifier") - } - - if sig.Type == crypto.SigTypeBLS || sig.Type == crypto.SigTypeSecp256k1 || sig.Type == crypto.SigTypeDelegated { - err = sigs.Verify(sig, d.Client, msg) - if err != nil { - return ErrBadProposal, xerrors.Errorf("invalid signature") - } - return Ok, nil - } - - // Add more types if required in Future - return ErrBadProposal, xerrors.Errorf("invalid signature type") -} - -func (d DataSource) Validate(db *harmonydb.DB) (ErrorCode, error) { +//func (d *Deal) ValidateSignature() (DealCode, error) { +// if len(d.Signature) == 0 { +// return ErrBadProposal, xerrors.Errorf("no signature") +// } +// +// sig := &crypto.Signature{} +// err := sig.UnmarshalBinary(d.Signature) +// if err != nil { +// return ErrBadProposal, xerrors.Errorf("invalid signature") +// } +// +// msg, err := d.Identifier.MarshalBinary() +// if err != nil { +// return ErrBadProposal, xerrors.Errorf("invalid identifier") +// } +// +// if sig.Type == crypto.SigTypeBLS || sig.Type == crypto.SigTypeSecp256k1 || sig.Type == crypto.SigTypeDelegated { +// err = sigs.Verify(sig, d.Client, msg) +// if err != nil { +// return ErrBadProposal, xerrors.Errorf("invalid signature") +// } +// return Ok, nil +// } +// +// // Add more types if required in Future +// return ErrBadProposal, xerrors.Errorf("invalid signature type") +//} + +func (d DataSource) Validate(db *harmonydb.DB) (DealCode, error) { err := ValidatePieceCID(d.PieceCID) if err != nil { @@ -296,7 +302,7 @@ func GetPieceInfo(c cid.Cid) (*PieceInfo, error) { }, nil } -func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (ErrorCode, error) { +func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { var nproducts int if d.DDOV1 != nil { nproducts++ @@ -611,7 +617,7 @@ func DBDealsToDeals(deals []*DBDeal) ([]*Deal, error) { } type ProviderDealRejectionInfo struct { - HTTPCode int + HTTPCode DealCode Reason string } @@ -625,11 +631,20 @@ type DealStatusResponse struct { ErrorMsg string `json:"error_msg"` } +type DealProductStatusResponse struct { + + // DDOV1 holds the DealStatusResponse for product "ddo_v1". + DDOV1 DealStatusResponse `json:"ddo_v1"` + + // PDPV1 represents the DealStatusResponse for the product pdp_v1. + PDPV1 DealStatusResponse `json:"pdp_v1"` +} + // DealStatus represents the status of a deal, including the HTTP code and an optional response detailing the deal's state and error message. type DealStatus struct { - // Response provides details about the deal's status, such as its current state and any associated error messages, if available. - Response *DealStatusResponse + // Response provides details about the deal's per product status, such as its current state and any associated error messages, if available. + Response *DealProductStatusResponse // HTTPCode represents the HTTP status code providing additional context about the deal status or possible errors. HTTPCode int @@ -685,7 +700,7 @@ func (dsh *DataSourceHttpPut) Name() DataSourceName { return DataSourceNamePut } -func IsDataSourceEnabled(db *harmonydb.DB, name DataSourceName) (ErrorCode, error) { +func IsDataSourceEnabled(db *harmonydb.DB, name DataSourceName) (DealCode, error) { var enabled bool err := db.QueryRow(context.Background(), `SELECT enabled FROM market_mk20_data_source WHERE name = $1`, name).Scan(&enabled) @@ -700,7 +715,7 @@ func IsDataSourceEnabled(db *harmonydb.DB, name DataSourceName) (ErrorCode, erro return Ok, nil } -func IsProductEnabled(db *harmonydb.DB, name ProductName) (ErrorCode, error) { +func IsProductEnabled(db *harmonydb.DB, name ProductName) (DealCode, error) { var enabled bool err := db.QueryRow(context.Background(), `SELECT enabled FROM market_mk20_products WHERE name = $1`, name).Scan(&enabled) @@ -728,8 +743,13 @@ type SupportedDataSources struct { Sources []string `json:"sources"` } -// StartUpload represents metadata for initiating an upload operation, containing the chunk size of the data to be uploaded. +// StartUpload represents metadata for initiating an upload operation. type StartUpload struct { + + // RawSize indicates the total size of the data to be uploaded in bytes. + RawSize uint64 `json:"raw_size"` + + // ChunkSize defines the size of each data chunk to be used during the upload process. ChunkSize int64 `json:"chunk_size"` } @@ -752,10 +772,10 @@ type UploadStatus struct { MissingChunks []int `json:"missing_chunks"` } -func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config) (*Deal, ErrorCode, error) { +func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config) (*Deal, DealCode, []ProductName, error) { ddeal, err := DealFromDB(ctx, db, id) if err != nil { - return nil, http.StatusInternalServerError, xerrors.Errorf("getting deal from DB: %w", err) + return nil, ErrServerInternalError, nil, xerrors.Errorf("getting deal from DB: %w", err) } // Run the following checks @@ -769,17 +789,158 @@ func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal ddeal.Data = deal.Data } + var newProducts []ProductName + if ddeal.Products.DDOV1 == nil || deal.Products.DDOV1 != nil { - return nil, ErrBadProposal, xerrors.Errorf("ddov1 update is not yet supported") + ddeal.Products.DDOV1 = deal.Products.DDOV1 + newProducts = append(newProducts, ProductNameDDOV1) } if ddeal.Products.RetrievalV1 == nil || deal.Products.RetrievalV1 != nil { ddeal.Products.RetrievalV1 = deal.Products.RetrievalV1 + newProducts = append(newProducts, ProductNameRetrievalV1) } code, err := ddeal.Validate(db, cfg) if err != nil { - return nil, code, xerrors.Errorf("validate deal: %w", err) + return nil, code, nil, xerrors.Errorf("validate deal: %w", err) + } + return ddeal, Ok, newProducts, nil +} + +func AuthenticateClient(db *harmonydb.DB, id, client string) (bool, error) { + var allowed bool + err := db.QueryRow(context.Background(), `SELECT EXISTS (SELECT 1 FROM market_mk20_deal WHERE id = $1 AND client = $2)`, id, client).Scan(&allowed) + if err != nil { + return false, xerrors.Errorf("querying client: %w", err) + } + return allowed, nil +} + +func clientAllowed(ctx context.Context, db *harmonydb.DB, client string) (bool, error) { + var allowed bool + err := db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_clients WHERE client = $1 AND IS allowed)`, client).Scan(&allowed) + if err != nil { + return false, xerrors.Errorf("querying client: %w", err) + } + return allowed, nil +} + +const Authprefix = "CurioAuth " + +// Auth verifies the custom authentication header by parsing its contents and validating the signature using the provided database connection. +func Auth(header, path string, db *harmonydb.DB) (bool, string, error) { + keyType, pubKey, sig, err := parseCustomAuth(header) + if err != nil { + return false, "", xerrors.Errorf("parsing auth header: %w", err) + } + return verifySignature(db, keyType, path, pubKey, sig) +} + +func parseCustomAuth(header string) (keyType string, pubKey, sig []byte, err error) { + + if !strings.HasPrefix(header, Authprefix) { + return "", nil, nil, errors.New("missing CustomAuth prefix") + } + + parts := strings.SplitN(strings.TrimPrefix(header, Authprefix), ":", 3) + if len(parts) != 3 { + return "", nil, nil, errors.New("invalid auth format") } - return ddeal, Ok, nil + + keyType = parts[0] + pubKey, err = base64.StdEncoding.DecodeString(parts[1]) + if err != nil { + return "", nil, nil, fmt.Errorf("invalid pubkey base64: %w", err) + } + + if len(pubKey) == 0 { + return "", nil, nil, fmt.Errorf("invalid pubkey") + } + + sig, err = base64.StdEncoding.DecodeString(parts[2]) + if err != nil { + return "", nil, nil, fmt.Errorf("invalid signature base64: %w", err) + } + + if len(sig) == 0 { + return "", nil, nil, fmt.Errorf("invalid signature") + } + + return keyType, pubKey, sig, nil +} + +func verifySignature(db *harmonydb.DB, keyType string, path string, pubKey, signature []byte) (bool, string, error) { + now := time.Now().Truncate(time.Minute) + minus1 := now.Add(-1 * time.Minute) + plus1 := now.Add(1 * time.Minute) + timeStamps := []time.Time{now, minus1, plus1} + var msgs [][32]byte + + for _, t := range timeStamps { + msgs = append(msgs, sha256.Sum256(bytes.Join([][]byte{pubKey, []byte(path), []byte(t.Format(time.RFC3339))}, []byte{}))) + } + + switch keyType { + case "ed25519": + if len(pubKey) != ed25519.PublicKeySize || len(signature) != ed25519.SignatureSize { + return false, "", errors.New("invalid ed25519 sizes") + } + keyStr, err := ED25519ToString(pubKey) + if err != nil { + return false, "", xerrors.Errorf("invalid ed25519 pubkey: %w", err) + } + for _, m := range msgs { + ok := ed25519.Verify(pubKey, m[:], signature) + if ok { + return true, keyStr, nil + } + } + return false, "", errors.New("invalid ed25519 signature") + + case "secp256k1", "bls", "delegated": + return verifyFilSignature(db, pubKey, signature, msgs) + default: + return false, "", fmt.Errorf("unsupported key type: %s", keyType) + } +} + +func verifyFilSignature(db *harmonydb.DB, pubKey, signature []byte, msgs [][32]byte) (bool, string, error) { + signs := &fcrypto.Signature{} + err := signs.UnmarshalBinary(signature) + if err != nil { + return false, "", xerrors.Errorf("invalid signature") + } + addr, err := address.NewFromBytes(pubKey) + if err != nil { + return false, "", xerrors.Errorf("invalid filecoin pubkey") + } + + allowed, err := clientAllowed(context.Background(), db, addr.String()) + if err != nil { + return false, "", xerrors.Errorf("checking client allowed: %w", err) + } + if !allowed { + return false, "", xerrors.Errorf("client not allowed") + } + + for _, m := range msgs { + err = sigs.Verify(signs, addr, m[:]) + if err == nil { + return true, addr.String(), nil + } + } + + return false, "", errors.New("invalid signature") +} + +func ED25519ToString(pubKey []byte) (string, error) { + if len(pubKey) != ed25519.PublicKeySize { + return "", errors.New("invalid ed25519 pubkey size") + } + return base58.FastBase58Encoding(pubKey), nil +} + +func StringToED25519(addr string) ([]byte, error) { + return base58.FastBase58Decoding(addr) } diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index 17275d81f..fa5689fb6 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -199,94 +199,137 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo // Update DB status of piece, deal, PDP comm, err = a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var refIDUsed bool // Update PoRep pipeline if deal.Products.DDOV1 != nil { - spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + var complete bool + err = tx.QueryRow(`SELECT ddo_v1->>'complete' FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) if err != nil { - return false, fmt.Errorf("getting provider ID: %w", err) + return false, fmt.Errorf("getting deal status: %w", err) } + if !complete { + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return false, fmt.Errorf("getting provider ID: %w", err) + } - var rev mk20.RetrievalV1 - if deal.Products.RetrievalV1 != nil { - rev = *deal.Products.RetrievalV1 - } + var rev mk20.RetrievalV1 + if deal.Products.RetrievalV1 != nil { + rev = *deal.Products.RetrievalV1 + } - ddo := deal.Products.DDOV1 - dealdata := deal.Data - dealID := deal.Identifier.String() + ddo := deal.Products.DDOV1 + dealdata := deal.Data + dealID := deal.Identifier.String() - var allocationID interface{} - if ddo.AllocationId != nil { - allocationID = *ddo.AllocationId - } else { - allocationID = nil - } + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } - aggregation := 0 - if dealdata.Format.Aggregate != nil { - aggregation = int(dealdata.Format.Aggregate.Type) - } + aggregation := 0 + if dealdata.Format.Aggregate != nil { + aggregation = int(dealdata.Format.Aggregate.Type) + } - if !pieceParked { - _, err = tx.Exec(`UPDATE parked_pieces SET + if !pieceParked { + _, err = tx.Exec(`UPDATE parked_pieces SET complete = TRUE WHERE id = $1 AND complete = false`, pieceRefID) - if err != nil { - return false, xerrors.Errorf("marking piece park as complete: %w", err) + if err != nil { + return false, xerrors.Errorf("marking piece park as complete: %w", err) + } } - } - pieceIDUrl := url.URL{ - Scheme: "pieceref", - Opaque: fmt.Sprintf("%d", pieceRefID), - } + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", pieceRefID), + } - n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( - id, sp_id, contract, client, piece_cid_v2, piece_cid, - piece_size, raw_size, url, offline, indexing, announce, - allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, - dealID, spid, ddo.ContractAddress, deal.Client.String(), pcid2.String(), pcid.String(), - psize, rawSize, pieceIDUrl.String(), false, rev.Indexing, rev.AnnouncePayload, - allocationID, ddo.Duration, aggregation, aggregation) - if err != nil { - return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) - } + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid_v2, piece_cid, + piece_size, raw_size, url, offline, indexing, announce, + allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, + dealID, spid, ddo.ContractAddress, deal.Client.String(), pcid2.String(), pcid.String(), + psize, rawSize, pieceIDUrl.String(), false, rev.Indexing, rev.AnnouncePayload, + allocationID, ddo.Duration, aggregation, aggregation) - _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) - if err != nil { - return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) - } + if err != nil { + return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } - _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) - if err != nil { - return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) - } + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) + } - _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) - if err != nil { - return false, xerrors.Errorf("deleting parked piece refs: %w", err) + _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) + } + + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) + if err != nil { + return false, xerrors.Errorf("deleting parked piece refs: %w", err) + } + + refIDUsed = true } } // Update PDP pipeline if deal.Products.PDPV1 != nil { - pdp := deal.Products.PDPV1 - n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + var complete bool + err = tx.QueryRow(`SELECT pdp_v1->>'complete' FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) + if err != nil { + return false, fmt.Errorf("getting deal status: %w", err) + } + if !complete { + pdp := deal.Products.PDPV1 + var newRefID int64 + if refIDUsed { + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, parkedPieceID, "/PUT").Scan(&newRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + } + } else { + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) - if err != nil { - return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + } + } + } } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 67f456368..df98e2501 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -102,7 +102,7 @@ func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { func (d *CurioStorageDealMarket) insertDDODealInPipeline(ctx context.Context) { var deals []string - rows, err := d.db.Query(ctx, `SELECT id from market_mk20_pipeline_waiting WHERE waiting_for_data = FALSE`) + rows, err := d.db.Query(ctx, `SELECT id from market_mk20_pipeline_waiting`) if err != nil { log.Errorf("querying mk20 pipeline waiting: %s", err) return From cb98d4813b8772d65d1e149cb362398450ab671c Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 21 Jul 2025 17:21:37 +0400 Subject: [PATCH 20/55] pdp prove test --- .github/workflows/ci.yml | 6 + itests/pdp_prove_test.go | 160 ++++++++++++++++++++++++ lib/proof/merkle_sha254_memtree.go | 45 ++++++- market/indexstore/indexstore.go | 4 + market/mk20/mk20.go | 25 ++-- market/mk20/mk20_upload.go | 25 ++-- market/mk20/utils.go | 8 ++ tasks/pdp/task_prove.go | 191 +++++++++-------------------- tasks/pdp/task_save_cache.go | 126 +++++++++++++------ 9 files changed, 392 insertions(+), 198 deletions(-) create mode 100644 itests/pdp_prove_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c6947896..6dbd18f6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,8 @@ jobs: target: "./itests/harmonydb_test.go" - name: test-itest-alertnow target: "./itests/alertnow_test.go" + - name: test-itest-pdp-prove + target: "./itests/pdp_prove_test.go" steps: - uses: actions/checkout@v4 @@ -311,6 +313,10 @@ jobs: run: go install github.com/hannahhoward/cbor-gen-for shell: bash + - name: Install swag cli + run: go install github.com/swaggo/swag/cmd/swag@latest + shell: bash + # - name: Install gotext # run: go install golang.org/x/text/cmd/gotext # shell: bash diff --git a/itests/pdp_prove_test.go b/itests/pdp_prove_test.go new file mode 100644 index 000000000..caea13af5 --- /dev/null +++ b/itests/pdp_prove_test.go @@ -0,0 +1,160 @@ +package itests + +import ( + "io" + "math/rand" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/proof" + _ "github.com/filecoin-project/curio/lib/proof" + "github.com/filecoin-project/curio/lib/testutils" + "github.com/filecoin-project/curio/pdp/contract" + "github.com/filecoin-project/curio/tasks/pdp" + + "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" +) + +// TestPDPProving verifies the functionality of generating and validating PDP proofs with a random file created in a temporary directory. +func TestPDPProving(t *testing.T) { + dir := t.TempDir() + + rawSize := int64(8323072) + //rawSize := int64(7 * 1024 * 1024 * 1024) + pieceSize := padreader.PaddedSize(uint64(rawSize)).Padded() + + // Create temporary file + fileStr, err := testutils.CreateRandomTmpFile(dir, rawSize) + require.NoError(t, err) + + defer os.Remove(fileStr) + + f, err := os.Open(fileStr) + require.NoError(t, err) + + stat, err := f.Stat() + require.NoError(t, err) + require.Equal(t, stat.Size(), rawSize) + + defer func() { + _ = f.Close() + }() + + t.Logf("File Size: %d", stat.Size()) + + // Total number of leafs + numberOfLeafs := pieceSize.Unpadded() / 32 + + // Do commP and save the snapshot layer + cp := pdp.NewCommPWithSizeForTest(uint64(rawSize)) + _, err = io.Copy(cp, f) + require.NoError(t, err) + + digest, psize, layerIdx, layer, err := cp.DigestWithSnapShot() + require.NoError(t, err) + + require.Equal(t, abi.PaddedPieceSize(psize), pieceSize) + + t.Logf("Digest: %x", digest) + t.Logf("PieceSize: %d", psize) + t.Logf("LayerIdx: %d", layerIdx) + t.Logf("Number of Nodes in snapshot layer: %d", len(layer)) + t.Logf("Total Number of Leafs: %d", numberOfLeafs) + + // Generate challenge leaf + rand.Seed(time.Now().UnixNano()) + challenge := int64(rand.Intn(int(numberOfLeafs))) + + t.Logf("Challenge: %d", challenge) + + // Calculate start leaf and snapshot leaf indexes + leavesPerNode := int64(1) << layerIdx + snapshotNodeIndex := challenge >> layerIdx + startLeaf := snapshotNodeIndex << layerIdx + t.Logf("Leaves per Node: %d", leavesPerNode) + t.Logf("Start Leaf: %d", startLeaf) + t.Logf("Snapshot Node Index: %d", snapshotNodeIndex) + + snapNode := layer[snapshotNodeIndex] + + // Convert tree-based leaf range to file-based offset/length + offset := int64(abi.PaddedPieceSize(startLeaf * 32).Unpadded()) + length := int64(abi.PaddedPieceSize(leavesPerNode * 32).Unpadded()) + + t.Logf("Offset: %d", offset) + t.Logf("Length: %d", length) + + // Compute padded size to build Merkle tree + subrootSize := padreader.PaddedSize(uint64(length)).Padded() + t.Logf("Subroot Size: %d", subrootSize) + + _, err = f.Seek(0, io.SeekStart) + require.NoError(t, err) + + dataReader := io.NewSectionReader(f, offset, length) + + _, err = f.Seek(offset, io.SeekStart) + require.NoError(t, err) + + fileRemaining := stat.Size() - offset + + t.Logf("File Remaining: %d", fileRemaining) + t.Logf("Is Padding: %t", fileRemaining < length) + + var data io.Reader + if fileRemaining < length { + data = io.MultiReader(dataReader, nullreader.NewNullReader(abi.UnpaddedPieceSize(int64(subrootSize.Unpadded())-fileRemaining))) + } else { + data = dataReader + } + + memtree, err := proof.BuildSha254Memtree(data, subrootSize.Unpadded()) + require.NoError(t, err) + + // Get challenge leaf in subTree + subTreeChallenge := challenge - startLeaf + + // Generate merkle proof for subTree + subTreeProof, err := proof.MemtreeProof(memtree, subTreeChallenge) + require.NoError(t, err) + + // Verify that subTree root is same as snapNode hash + require.Equal(t, subTreeProof.Root, snapNode.Hash) + + // Arrange snapshot layer into a byte array + var layerBytes []byte + for _, node := range layer { + layerBytes = append(layerBytes, node.Hash[:]...) + } + + t.Logf("Layer Bytes: %d", len(layerBytes)) + + // Create subTree from snapshot to commP (root) + mtree, err := proof.BuildSha254MemtreeFromSnapshot(layerBytes) + require.NoError(t, err) + + // Generate merkle proof from snapShot node to commP + proofs, err := proof.MemtreeProof(mtree, snapshotNodeIndex) + require.NoError(t, err) + + var digest32 [32]byte + copy(digest32[:], digest[:]) + + // verify that root and commP match + require.Equal(t, proofs.Root, digest32) + rd := proofs.Root + + out := contract.PDPVerifierProof{ + Leaf: subTreeProof.Leaf, + Proof: append(subTreeProof.Proof, proofs.Proof...), + } + + verified := pdp.Verify(out, rd, uint64(challenge)) + require.True(t, verified) +} diff --git a/lib/proof/merkle_sha254_memtree.go b/lib/proof/merkle_sha254_memtree.go index fec3daecd..009d10159 100644 --- a/lib/proof/merkle_sha254_memtree.go +++ b/lib/proof/merkle_sha254_memtree.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lotus/storage/sealer/fr32" ) -const MaxMemtreeSize = 256 << 20 +const MaxMemtreeSize = 1 << 30 // BuildSha254Memtree builds a sha256 memtree from the input data // Returned slice should be released to the pool after use @@ -72,3 +72,46 @@ func ComputeBinShaParent(left, right [NODE_SIZE]byte) [NODE_SIZE]byte { out[NODE_SIZE-1] &= 0x3F return out } + +func BuildSha254MemtreeFromSnapshot(data []byte) ([]byte, error) { + size := abi.PaddedPieceSize(len(data)) + if size > MaxMemtreeSize { + return nil, xerrors.Errorf("piece too large for memtree: %d", size) + } + + nLeaves := int64(size) / NODE_SIZE + totalNodes, levelSizes := computeTotalNodes(nLeaves, 2) + memtreeBuf := pool.Get(int(totalNodes * NODE_SIZE)) + + copy(memtreeBuf[:len(data)], data) + + d := sha256.New() + + levelStarts := make([]int64, len(levelSizes)) + levelStarts[0] = 0 + for i := 1; i < len(levelSizes); i++ { + levelStarts[i] = levelStarts[i-1] + levelSizes[i-1]*NODE_SIZE + } + + for level := 1; level < len(levelSizes); level++ { + levelNodes := levelSizes[level] + prevLevelStart := levelStarts[level-1] + currLevelStart := levelStarts[level] + + for i := int64(0); i < levelNodes; i++ { + leftOffset := prevLevelStart + (2*i)*NODE_SIZE + + d.Reset() + d.Write(memtreeBuf[leftOffset : leftOffset+(NODE_SIZE*2)]) + + outOffset := currLevelStart + i*NODE_SIZE + // sum calls append, so we give it a zero len slice at the correct offset + d.Sum(memtreeBuf[outOffset:outOffset]) + + // set top bits to 00 + memtreeBuf[outOffset+NODE_SIZE-1] &= 0x3F + } + } + + return memtreeBuf, nil +} diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index 7d1425a3e..7a23b1a25 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math/rand" + "sort" "strconv" "strings" "time" @@ -647,6 +648,9 @@ func (i *IndexStore) GetPDPLayer(ctx context.Context, pieceCidV2 cid.Cid) ([]Nod if err := iter.Close(); err != nil { return nil, xerrors.Errorf("iterating PDP cache layer (P:0x%02x): %w", pieceCidV2.Bytes(), err) } + sort.Slice(layer, func(i, j int) bool { + return layer[i].Index < layer[j].Index + }) return layer, nil } diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index cf0cb3c76..300663fe2 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" verifreg9 "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/ffi" @@ -131,7 +132,15 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti log.Debugw("deal validated", "deal", deal.Identifier.String()) if deal.Products.DDOV1 != nil { - return m.processDDODeal(ctx, deal) + // TODO: Remove this check once DDO market is done + if build.BuildType == build.Build2k || build.BuildType == build.BuildDebug { + return m.processDDODeal(ctx, deal) + } + log.Errorw("DDOV1 is not supported yet", "deal", deal.Identifier.String()) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrUnsupportedProduct, + Reason: "DDOV1 is not supported yet", + } } return m.processPDPDeal(ctx, deal) @@ -678,20 +687,8 @@ func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, w http.ResponseWriter) { ctx := context.Background() - allowed, err := AuthenticateClient(m.DB, deal.Identifier.String(), deal.Client.String()) - if err != nil { - log.Errorw("deal rejected", "deal", deal, "error", err) - http.Error(w, "", int(ErrServerInternalError)) - return - } - if !allowed { - log.Errorw("deal rejected as client is not authorized", "deal", deal) - http.Error(w, "client not authorized", int(ErrUnAuthorized)) - return - } - var exists bool - err = m.DB.QueryRow(ctx, `SELECT EXISTS ( + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 FROM market_mk20_deal WHERE id = $1)`, id.String()).Scan(&exists) diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index c206e0a7c..e812f5418 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -424,18 +424,6 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { ctx := context.Background() - if deal != nil { - allow, err := AuthenticateClient(m.DB, id.String(), deal.Client.String()) - if err != nil { - log.Errorw("failed to authenticate client", "deal", id, "error", err) - http.Error(w, "", int(ErrServerInternalError)) - return - } - if !allow { - http.Error(w, "client is not authorized to finalize deal", http.StatusUnauthorized) - return - } - } var exists bool err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 @@ -453,6 +441,19 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri return } + ddeal, err := DealFromDB(ctx, m.DB, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + + if ddeal.Data == nil && deal == nil { + log.Errorw("cannot finalize deal with missing data source", "deal", id) + http.Error(w, "cannot finalize deal with missing data source", int(ErrBadProposal)) + return + } + if deal != nil { // This is a deal where DataSource was not set - we should update the deal code, ndeal, _, err := m.updateDealDetails(id, deal) diff --git a/market/mk20/utils.go b/market/mk20/utils.go index b9d2c9e84..c1e4690e7 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -785,6 +785,14 @@ func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal // If PDPv1 is defined by DDOV1 is not, then allow updating it // If DDOV1 is defined then don't allow PDPv1 yet + // TODO: Remove this once DDO is live + if ddeal.Products.PDPV1 != nil { + if ddeal.Data == nil { + ddeal.Data = deal.Data + } + return ddeal, Ok, nil, nil + } + if ddeal.Data == nil { ddeal.Data = deal.Data } diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index 0cb9958a6..a6a4ab5f6 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "io" + "math" "math/big" "sync/atomic" @@ -383,117 +384,6 @@ func padTo32Bytes(b []byte) []byte { return padded } -func (p *ProveTask) genSubrootMemtree( - ctx context.Context, - pieceCidV2 cid.Cid, - challengedLeafIndex int64, - savedLayer int, -) ([]byte, error) { - // Calculate which snapshot node covers this challenged leaf - leavesPerNode := int64(1) << savedLayer - snapshotNodeIndex := challengedLeafIndex >> savedLayer - startLeaf := snapshotNodeIndex << savedLayer - - // Convert tree-based leaf range to file-based offset/length - offset := startLeaf * inputBytesPerLeaf - length := leavesPerNode * inputBytesPerLeaf - - // Compute padded size to build Merkle tree (must match what BuildSha254Memtree expects) - subrootSize := padreader.PaddedSize(uint64(length)).Padded() - if subrootSize > proof.MaxMemtreeSize { - return nil, xerrors.Errorf("subroot size exceeds maximum: %d", subrootSize) - } - - // Get original file reader - reader, reportedSize, err := p.cpr.GetSharedPieceReader(ctx, pieceCidV2) - if err != nil { - return nil, xerrors.Errorf("failed to get reader: %w", err) - } - defer reader.Close() - - if offset > int64(reportedSize) { - // The entire requested range is beyond file size ā†’ pure padding - // This should never happen - //TODO: Maybe put a panic here? - paddingOnly := nullreader.NewNullReader(abi.UnpaddedPieceSize(length)) - return proof.BuildSha254Memtree(paddingOnly, subrootSize.Unpadded()) - } - - _, err = reader.Seek(offset, io.SeekStart) - if err != nil { - return nil, xerrors.Errorf("seek to offset %d failed: %w", offset, err) - } - - // Read up to file limit - var data io.Reader - fileRemaining := int64(reportedSize) - offset - if fileRemaining < length { - data = io.MultiReader(io.LimitReader(reader, fileRemaining), nullreader.NewNullReader(abi.UnpaddedPieceSize(int64(subrootSize.Unpadded())-fileRemaining))) - } else { - data = io.LimitReader(reader, length) - } - - // Build Merkle tree from padded input - return proof.BuildSha254Memtree(data, subrootSize.Unpadded()) -} - -func GenerateProofToRootFromSnapshot( - snapshotLayer int, - snapshotIndex int64, - snapshotHash [32]byte, - snapshotNodes []indexstore.NodeDigest, -) ([][32]byte, [32]byte, error) { - snapMap := make(map[int64][32]byte) - for _, n := range snapshotNodes { - if n.Layer != snapshotLayer { - continue // ignore other layers if present - } - snapMap[n.Index] = n.Hash - } - - proof := make([][32]byte, 0) - currentHash := snapshotHash - currentIndex := snapshotIndex - hasher := sha256.New() - - for level := snapshotLayer + 1; ; level++ { - siblingIndex := currentIndex ^ 1 - - siblingHash, exists := snapMap[siblingIndex] - if !exists { - // Padding if sibling missing - siblingHash = currentHash - } - - // Add sibling to proof - proof = append(proof, siblingHash) - - // Compute parent - hasher.Reset() - if currentIndex%2 == 0 { - hasher.Write(currentHash[:]) - hasher.Write(siblingHash[:]) - } else { - hasher.Write(siblingHash[:]) - hasher.Write(currentHash[:]) - } - sum := hasher.Sum(nil) - var parent [32]byte - copy(parent[:], sum) - parent[31] &= 0x3F - - currentHash = parent - currentIndex = currentIndex >> 1 - - // stop when we reach the root (single node at level) - if len(snapMap) <= 1 && currentIndex == 0 { - break - } - } - - return proof, currentHash, nil -} - func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int64, challengedLeaf int64) (contract.PDPVerifierProof, error) { //const arity = 2 @@ -547,10 +437,15 @@ func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int6 rootDigest = mProof.Root } else { - layerIdx := snapshotLayerIndex(pi.RawSize) - cacheIdx := challengedLeaf >> layerIdx + //Calculate layer L such that 127 * 2^L >= targetReadSize + //ā†’ 2^L >= targetReadSize / 32 + ratio := float64(4161536) / 32 + layerIdx := int(math.Ceil(math.Log2(ratio))) - has, node, err := p.idx.GetPDPNode(ctx, pcid, cacheIdx) + leavesPerNode := int64(1) << layerIdx + snapshotNodeIndex := challengedLeaf >> layerIdx + + has, node, err := p.idx.GetPDPNode(ctx, pcid, snapshotNodeIndex) if err != nil { return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get node: %w", err) } @@ -561,26 +456,45 @@ func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int6 panic("implement me") } - log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf, "layerIdx", layerIdx, "cacheIdx", cacheIdx, "node", node) + log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf, "layerIdx", layerIdx, "snapshotNodeIndex", snapshotNodeIndex, "node", node) if node.Layer != layerIdx { return contract.PDPVerifierProof{}, xerrors.Errorf("node layer mismatch: %d != %d", node.Layer, layerIdx) } - // build subroot memtree - memtree, err := p.genSubrootMemtree(ctx, pcid, challengedLeaf, layerIdx) + startLeaf := snapshotNodeIndex << layerIdx + // Convert tree-based leaf range to file-based offset/length + offset := int64(abi.PaddedPieceSize(startLeaf * 32).Unpadded()) + length := int64(abi.PaddedPieceSize(leavesPerNode * 32).Unpadded()) + + // Compute padded size to build Merkle tree + subrootSize := padreader.PaddedSize(uint64(length)).Padded() + + // Get original file reader + reader, reportedSize, err := p.cpr.GetSharedPieceReader(ctx, pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate subroot memtree: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get reader: %w", err) } + defer reader.Close() - /* - type RawMerkleProof struct { - Leaf [32]byte - Proof [][32]byte - Root [32]byte - } - */ - subTreeProof, err := proof.MemtreeProof(memtree, challengedLeaf) + fileRemaining := int64(reportedSize) - offset + + var data io.Reader + if fileRemaining < length { + data = io.MultiReader(reader, nullreader.NewNullReader(abi.UnpaddedPieceSize(int64(subrootSize.Unpadded())-fileRemaining))) + } else { + data = reader + } + + memtree, err := proof.BuildSha254Memtree(data, subrootSize.Unpadded()) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree: %w", err) + } + + // Get challenge leaf in subTree + subTreeChallenge := challengedLeaf - startLeaf + + subTreeProof, err := proof.MemtreeProof(memtree, subTreeChallenge) if err != nil { return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate sub tree proof: %w", err) } @@ -597,9 +511,22 @@ func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int6 return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get layer nodes: %w", err) } - proofs, rd, err := GenerateProofToRootFromSnapshot(node.Layer, node.Index, node.Hash, layerNodes) + // Arrange snapshot layer into a byte array + var layerBytes []byte + for _, n := range layerNodes { + layerBytes = append(layerBytes, n.Hash[:]...) + } + + // Create subTree from snapshot to commP (root) + mtree, err := proof.BuildSha254MemtreeFromSnapshot(layerBytes) + if err != nil { + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree from snapshot: %w", err) + } + + // Generate merkle proof from snapShot node to commP + proofs, err := proof.MemtreeProof(mtree, snapshotNodeIndex) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate proof to root: %w", err) + return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) } com, err := commcidv2.CommPFromPCidV2(pcid) @@ -608,16 +535,16 @@ func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int6 } // Verify proof with original root - if [32]byte(com.Digest()) != rd { - return contract.PDPVerifierProof{}, xerrors.Errorf("root digest mismatch: %x != %x", com.Digest(), rd) + if [32]byte(com.Digest()) != proofs.Root { + return contract.PDPVerifierProof{}, xerrors.Errorf("root digest mismatch: %x != %x", com.Digest(), proofs.Root) } out = contract.PDPVerifierProof{ Leaf: subTreeProof.Leaf, - Proof: append([][32]byte{subTreeProof.Root}, proofs...), + Proof: append(subTreeProof.Proof, proofs.Proof...), } - rootDigest = rd + rootDigest = proofs.Root } if !Verify(out, rootDigest, uint64(challengedLeaf)) { diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go index 63b2c91b7..445dafb36 100644 --- a/tasks/pdp/task_save_cache.go +++ b/tasks/pdp/task_save_cache.go @@ -4,6 +4,7 @@ import ( "context" "hash" "io" + "math" "math/bits" "sync" "time" @@ -12,6 +13,8 @@ import ( sha256simd "github.com/minio/sha256-simd" "golang.org/x/xerrors" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" @@ -46,11 +49,10 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) ID string `db:"id"` PieceCid string `db:"piece_cid"` ProofSetID int64 `db:"proof_set_id"` - ExtraData []byte `db:"extra_data"` PieceRef string `db:"piece_ref"` } - err = t.db.Select(ctx, &saveCaches, `SELECT id, piece_cid, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE save_cache_task_id = $1 AND after_save_cache = FALSE`, taskID) + err = t.db.Select(ctx, &saveCaches, `SELECT id, piece_cid_v2, proof_set_id, piece_ref FROM pdp_pipeline WHERE save_cache_task_id = $1 AND after_save_cache = FALSE`, taskID) if err != nil { return false, xerrors.Errorf("failed to select addRoot: %w", err) } @@ -115,7 +117,7 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) leafs[i] = indexstore.NodeDigest{ Layer: lidx, Hash: s.Hash, - Index: s.Index, + Index: int64(i), } } @@ -201,22 +203,24 @@ var _ harmonytask.TaskInterface = &TaskSavePDPCache{} // accept Write()s without further initialization. type Calc struct { state - mu sync.Mutex + mu sync.Mutex + snapShotLayerIdx int + snapshotNodes []NodeDigest + snapshotNodesMu sync.Mutex + expectedNodeCount int + maxLayer uint + maxlayerMU sync.Mutex } type state struct { - quadsEnqueued uint64 - layerQueues [MaxLayers + 2]chan []byte // one extra layer for the initial leaves, one more for the dummy never-to-use channel - resultCommP chan []byte - buffer []byte - size uint64 - snapShotLayerIdx int - snapshotNodes []NodeDigest - snapshotNodesMu sync.Mutex + quadsEnqueued uint64 + layerQueues [MaxLayers + 2]chan []byte // one extra layer for the initial leaves, one more for the dummy never-to-use channel + resultCommP chan []byte + buffer []byte + size uint64 } type NodeDigest struct { - Index int64 // logical index at that layer - Hash [32]byte // 32 bytes + Hash [32]byte // 32 bytes } var _ hash.Hash = &Calc{} // make sure we are hash.Hash compliant @@ -472,7 +476,7 @@ func (cp *Calc) addLayer(myIdx uint) { panic("addLayer called more than once with identical idx argument") } cp.layerQueues[myIdx+1] = make(chan []byte, layerQueueDepth) - + collectSnapshot := int(myIdx) == cp.snapShotLayerIdx-1 go func() { var twinHold []byte @@ -491,7 +495,7 @@ func (cp *Calc) addLayer(myIdx uint) { if twinHold != nil { copy(twinHold[32:64], stackedNulPadding[myIdx]) - cp.hashSlab254(0, twinHold[0:64]) + cp.hashSlab254(0, collectSnapshot, twinHold[0:64]) cp.layerQueues[myIdx+1] <- twinHold[0:64:64] } @@ -504,12 +508,12 @@ func (cp *Calc) addLayer(myIdx uint) { switch { case len(slab) > 1<<(5+myIdx): - cp.hashSlab254(myIdx, slab) + cp.hashSlab254(myIdx, collectSnapshot, slab) cp.layerQueues[myIdx+1] <- slab pushedWork = true case twinHold != nil: copy(twinHold[32:64], slab[0:32]) - cp.hashSlab254(0, twinHold[0:64]) + cp.hashSlab254(0, collectSnapshot, twinHold[0:64]) cp.layerQueues[myIdx+1] <- twinHold[0:32:64] pushedWork = true twinHold = nil @@ -528,10 +532,13 @@ func (cp *Calc) addLayer(myIdx uint) { }() } -func (cp *Calc) hashSlab254(layerIdx uint, slab []byte) { +func (cp *Calc) hashSlab254(layerIdx uint, collectSnapshot bool, slab []byte) { h := shaPool.Get().(hash.Hash) - collectSnapshot := int(layerIdx) == cp.snapShotLayerIdx - + cp.maxlayerMU.Lock() + defer cp.maxlayerMU.Unlock() + if cp.maxLayer < layerIdx { + cp.maxLayer = layerIdx + } stride := 1 << (5 + layerIdx) for i := 0; len(slab) > i+stride; i += 2 * stride { h.Reset() @@ -544,8 +551,7 @@ func (cp *Calc) hashSlab254(layerIdx uint, slab []byte) { copy(d, slab[i:i+32]) cp.snapshotNodesMu.Lock() cp.snapshotNodes = append(cp.snapshotNodes, NodeDigest{ - Index: int64(i / 32), // logical index at this layer - Hash: [32]byte(d), + Hash: [32]byte(d), }) cp.snapshotNodesMu.Unlock() } @@ -558,7 +564,7 @@ func NewCommPWithSize(size uint64) *Calc { c := new(Calc) c.state.size = size - c.snapShotLayerIdx = snapshotLayerIndex(size) + c.snapshotLayerIndex(size, false) return c } @@ -568,25 +574,43 @@ const ( inputBytesPerLeaf = 127 // raw input bytes that become one 32-byte leaf ) -func snapshotLayerIndex(size uint64) int { +func (cp *Calc) snapshotLayerIndex(size uint64, test bool) { if size == 0 { panic("size must be > 0") } - // Total number of leaves, each representing 127 bytes of input - numLeaves := size / inputBytesPerLeaf - - // What is the top layer index (leaf layer = 0) - leafLayer := bits.Len64(numLeaves - 1) // ceil(log2) + // Calculate padded piece size + padded := padreader.PaddedSize(size).Padded() + + // Calculate number of leaf nodes (each covers 128 bytes) + numLeaves := uint64(padded) / 32 + + // Total tree height: log2(numLeaves) + treeHeight := bits.Len64(numLeaves - 1) + + //Calculate layer L such that 127 * 2^L >= targetReadSize + //ā†’ 2^L >= targetReadSize / 32 + //ratio := float64(1040384) / 32 + testRatio := float64(2032) / 32 + ProdRatio := float64(4161536) / 32 + var layer int + if test { + layer = int(math.Ceil(math.Log2(testRatio))) + } else { + layer = int(math.Ceil(math.Log2(ProdRatio))) + } - // At layer `i`, each node spans 2^i leaves - // Each leaf = 127 bytes ā‡’ node at layer i = 127 * 2^i - // Want: 127 * 2^i ā‰ˆ 4 MiB - // So: i = log2(4 MiB / 127) - targetSpanLeaves := targetReadSize / inputBytesPerLeaf - layerDelta := bits.Len64(uint64(targetSpanLeaves - 1)) + // Clamp within tree bounds + cp.snapShotLayerIdx = layer + if layer < 0 { + cp.snapShotLayerIdx = 0 + } + if layer > treeHeight { + cp.snapShotLayerIdx = treeHeight + } - return leafLayer - layerDelta + expectedNodes := numLeaves >> uint(cp.snapShotLayerIdx) + cp.expectedNodeCount = int(expectedNodes) } func (cp *Calc) DigestWithSnapShot() ([]byte, uint64, int, []NodeDigest, error) { @@ -598,7 +622,31 @@ func (cp *Calc) DigestWithSnapShot() ([]byte, uint64, int, []NodeDigest, error) cp.snapshotNodesMu.Lock() defer cp.snapshotNodesMu.Unlock() - out := make([]NodeDigest, len(cp.snapshotNodes)) - copy(out, cp.snapshotNodes) + // Make output array of expected length + out := make([]NodeDigest, cp.expectedNodeCount) + + // Copy snapShot nodes to output + copied := copy(out[:len(cp.snapshotNodes)], cp.snapshotNodes) + + // Fill remaining nodes with zeroPadding + if copied != cp.expectedNodeCount { + count := cp.expectedNodeCount - copied + var h [32]byte + copy(h[:], stackedNulPadding[cp.snapShotLayerIdx]) + out = append(out, make([]NodeDigest, count)...) + for i := copied; i < len(out); i++ { + out[i].Hash = h + } + } + return commp, paddedPieceSize, cp.snapShotLayerIdx, out, nil } + +func NewCommPWithSizeForTest(size uint64) *Calc { + c := new(Calc) + c.state.size = size + + c.snapshotLayerIndex(size, true) + + return c +} From c72541af64014c63bd65c0567485b61b4efb3ed5 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Mon, 21 Jul 2025 21:23:49 +0400 Subject: [PATCH 21/55] deleteRoot, deleteProofSet --- .github/workflows/ci.yml | 2 +- cmd/curio/tasks/tasks.go | 3 + .../harmonydb/sql/20250505-market_mk20.sql | 28 ++- market/mk20/http/docs.go | 11 ++ market/mk20/http/swagger.json | 11 ++ market/mk20/http/swagger.yaml | 9 + market/mk20/mk20.go | 34 ++++ market/mk20/pdp_v1.go | 76 ++++++-- tasks/pdp/proofset_create_watch.go | 4 +- tasks/pdp/proofset_delete_root_watch.go | 138 +++++++++++++ tasks/pdp/proofset_delete_watch.go | 161 ++++++++++++++++ tasks/pdp/task_delete_root.go | 182 ++++++++++++++++++ tasks/pdp/task_delete_rootset.go | 175 +++++++++++++++++ tasks/pdp/task_save_cache.go | 9 +- 14 files changed, 814 insertions(+), 29 deletions(-) create mode 100644 tasks/pdp/proofset_delete_root_watch.go create mode 100644 tasks/pdp/proofset_delete_watch.go create mode 100644 tasks/pdp/task_delete_root.go create mode 100644 tasks/pdp/task_delete_rootset.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dbd18f6f..76e1fdfbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -314,7 +314,7 @@ jobs: shell: bash - name: Install swag cli - run: go install github.com/swaggo/swag/cmd/swag@latest + run: go install github.com/swaggo/swag/cmd/swag@v1.16.4 shell: bash # - name: Install gotext diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 23786c6bf..b744f6432 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -288,6 +288,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan pdp.NewWatcherCreate(db, must.One(dependencies.EthClient.Val()), chainSched) pdp.NewWatcherRootAdd(db, must.One(dependencies.EthClient.Val()), chainSched) + pdp.NewWatcherDelete(db, must.One(dependencies.EthClient.Val()), chainSched) + pdp.NewPDPTaskDeleteRoot(db, es, must.One(dependencies.EthClient.Val())) + pdp.NewWatcherRootDelete(db, must.One(dependencies.EthClient.Val()), chainSched) pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) pdpCache := pdp.NewTaskSavePDPCache(db, dependencies.CachedPieceReader, iStore) diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 8f1376162..cbc0a1951 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -378,6 +378,8 @@ CREATE TABLE pdp_proof_set ( create_deal_id TEXT NOT NULL, -- mk20 deal ID for creating this proofset create_message_hash TEXT NOT NULL, + removed BOOLEAN DEFAULT FALSE, + remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this proofset remove_message_hash TEXT DEFAULT NULL, @@ -391,8 +393,31 @@ CREATE TABLE pdp_proof_set_create ( record_keeper TEXT NOT NULL, extra_data BYTEA, + + task_id BIGINT DEFAULT NULL, + tx_hash TEXT DEFAULT NULL +); + +CREATE TABLE pdp_proof_set_delete ( + id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response + client TEXT NOT NULL, + + set_id BIGINT NOT NULL, + extra_data BYTEA, + task_id BIGINT DEFAULT NULL, + tx_hash TEXT DEFAULT NULL +); + +CREATE TABLE pdp_delete_root ( + id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response + client TEXT NOT NULL, + + set_id BIGINT NOT NULL, + roots BIGINT[] NOT NULL, + extra_data BYTEA, + task_id BIGINT DEFAULT NULL, tx_hash TEXT DEFAULT NULL ); @@ -413,6 +438,7 @@ CREATE TABLE pdp_proofset_root ( add_message_hash TEXT NOT NULL, add_message_index BIGINT NOT NULL, -- index of root in the add message + removed BOOLEAN DEFAULT FALSE, remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this root from proofset remove_message_hash TEXT DEFAULT NULL, remove_message_index BIGINT DEFAULT NULL, @@ -450,7 +476,7 @@ CREATE TABLE pdp_pipeline ( add_root_task_id BIGINT DEFAULT NULL, after_add_root BOOLEAN DEFAULT FALSE, - add_message_hash TEXT NOT NULL, + add_message_hash TEXT, add_message_index BIGINT NOT NULL DEFAULT 0, -- index of root in the add message after_add_root_msg BOOLEAN DEFAULT FALSE, diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 54add3928..9f4d5646a 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -930,6 +930,17 @@ const docTemplate = `{ "proof_set_id": { "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", "type": "integer" + }, + "record_keeper": { + "description": "RecordKeeper specifies the record keeper contract address for the new PDP proofset.", + "type": "string" + }, + "root_ids": { + "description": "RootIDs is a list of root ids in a proof set.", + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 9dbf0e813..3a7d62b70 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -921,6 +921,17 @@ "proof_set_id": { "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", "type": "integer" + }, + "record_keeper": { + "description": "RecordKeeper specifies the record keeper contract address for the new PDP proofset.", + "type": "string" + }, + "root_ids": { + "description": "RootIDs is a list of root ids in a proof set.", + "type": "array", + "items": { + "type": "integer" + } } } }, diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index 113d6a758..47e98df71 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -243,6 +243,15 @@ definitions: description: ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. type: integer + record_keeper: + description: RecordKeeper specifies the record keeper contract address for + the new PDP proofset. + type: string + root_ids: + description: RootIDs is a list of root ids in a proof set. + items: + type: integer + type: array type: object mk20.PieceDataFormat: properties: diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 300663fe2..614486cc0 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -372,6 +372,40 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } } + pdp := deal.Products.PDPV1 + if pdp.CreateProofSet { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_proof_set_create (id, client, record_keeper, extra_data) VALUES ($1, $2, $3, $4)`, + deal.Identifier.String(), deal.Client.String(), pdp.RecordKeeper, pdp.ExtraData) + if err != nil { + return false, xerrors.Errorf("inserting PDP proof set create: %w", err) + } + if n != 1 { + return false, fmt.Errorf("expected 1 row to be updated, got %d", n) + } + } + + if pdp.DeleteProofSet { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_proof_set_delete (id, client, set_id, extra_data) VALUES ($1, $2, $3, $4)`, + deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.ExtraData) + if err != nil { + return false, xerrors.Errorf("inserting PDP proof set delete: %w", err) + } + if n != 1 { + return false, fmt.Errorf("expected 1 row to be updated, got %d", n) + } + } + + if pdp.DeleteRoot { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_delete_root (id, client, set_id, roots, extra_data) VALUES ($1, $2, $3, $4, $5)`, + deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.ExtraData) + if err != nil { + return false, xerrors.Errorf("inserting PDP delete root: %w", err) + } + if n != 1 { + return false, fmt.Errorf("expected 1 row to be updated, got %d", n) + } + } + return true, nil }, harmonydb.OptionRetry()) if err != nil { diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 17e4a5a06..bd968a80f 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -3,6 +3,7 @@ package mk20 import ( "context" + "github.com/ethereum/go-ethereum/common" "golang.org/x/xerrors" "github.com/filecoin-project/curio/deps/config" @@ -27,6 +28,12 @@ type PDPV1 struct { // ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. ProofSetID *uint64 `json:"proof_set_id"` + // RecordKeeper specifies the record keeper contract address for the new PDP proofset. + RecordKeeper string `json:"record_keeper"` + + // RootIDs is a list of root ids in a proof set. + RootIDs []uint64 `json:"root_ids"` + // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. ExtraData []byte `json:"extra_data"` } @@ -41,20 +48,49 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return ErrBadProposal, xerrors.Errorf("deal must have one of the following flags set: create_proof_set, delete_proof_set, add_root, delete_root") } - if p.CreateProofSet && p.ProofSetID != nil { - return ErrBadProposal, xerrors.Errorf("create_proof_set cannot be set with proof_set_id") + if p.CreateProofSet { + if p.ProofSetID != nil { + return ErrBadProposal, xerrors.Errorf("create_proof_set cannot be set with proof_set_id") + } + if p.RecordKeeper == "" { + return ErrBadProposal, xerrors.Errorf("record_keeper must be defined for create_proof_set") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for create_proof_set") + } + if !common.IsHexAddress(p.RecordKeeper) { + return ErrBadProposal, xerrors.Errorf("record_keeper must be a valid address") + } } - if p.DeleteProofSet && p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") + if p.DeleteProofSet { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_proof_set") + } } - if p.AddRoot && p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") + if p.AddRoot { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for add_root") + } } - if p.DeleteRoot && p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") + if p.DeleteRoot { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") + } + if len(p.ExtraData) == 0 { + return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") + } + if len(p.RootIDs) == 0 { + return ErrBadProposal, xerrors.Errorf("root_ids must be defined for delete_proof_set") + } } // Only 1 action is allowed per deal @@ -62,18 +98,12 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return ErrBadProposal, xerrors.Errorf("only one action is allowed per deal") } - if p.CreateProofSet { - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for create_proof_set") - } - } - ctx := context.Background() if p.DeleteProofSet { pid := *p.ProofSetID var exists bool - err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) if err != nil { return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } @@ -88,7 +118,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er if p.AddRoot { pid := *p.ProofSetID var exists bool - err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) if err != nil { return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } @@ -103,9 +133,19 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er if p.DeleteRoot { pid := *p.ProofSetID var exists bool - err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND remove_deal_id IS NULL)`, pid).Scan(&exists) + err := db.QueryRow(ctx, `SELECT COUNT(*) = cardinality($2::BIGINT[]) AS all_exist_and_active + FROM pdp_proofset_root r + JOIN pdp_proof_set s ON r.proofset = s.id + WHERE r.proofset = $1 + AND r.root = ANY($2) + AND r.removed = FALSE + AND s.removed = FALSE; + )`, pid, p.RootIDs).Scan(&exists) if err != nil { - return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if proofset and roots exists: %w", err) + } + if !exists { + return ErrBadProposal, xerrors.Errorf("proofset or one of the roots does not exist") } if len(p.ExtraData) == 0 { return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") diff --git a/tasks/pdp/proofset_create_watch.go b/tasks/pdp/proofset_create_watch.go index 525201dda..908190b1c 100644 --- a/tasks/pdp/proofset_create_watch.go +++ b/tasks/pdp/proofset_create_watch.go @@ -37,7 +37,7 @@ func NewWatcherCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chains } func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { - // Query for pdp_proofset_creates entries where ok = TRUE and proofset_created = FALSE + // Query for pdp_proof_set_create entries tx_hash is NOT NULL var proofSetCreates []ProofSetCreate err := db.Select(ctx, &proofSetCreates, ` @@ -153,7 +153,7 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr n, err = tx.Exec(`UPDATE market_mk20_deal SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) - WHERE id = $1;`, "Transaction failed", psc.ID) + WHERE id = $1;`, psc.ID) if err != nil { return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) } diff --git a/tasks/pdp/proofset_delete_root_watch.go b/tasks/pdp/proofset_delete_root_watch.go new file mode 100644 index 000000000..3a26bbcb4 --- /dev/null +++ b/tasks/pdp/proofset_delete_root_watch.go @@ -0,0 +1,138 @@ +package pdp + +import ( + "context" + "encoding/json" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/chainsched" + + chainTypes "github.com/filecoin-project/lotus/chain/types" +) + +type ProofSetRootDelete struct { + ID string `db:"id"` + ProofSet uint64 `db:"set_id"` + Roots int64 `db:"roots"` + Hash string `db:"tx_hash"` +} + +func NewWatcherRootDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { + if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { + err := processPendingProofSetRootDeletes(ctx, db, ethClient) + if err != nil { + log.Warnf("Failed to process pending proof set creates: %v", err) + } + return nil + }); err != nil { + panic(err) + } +} + +func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { + var proofSetRootDeletes []ProofSetRootDelete + err := db.Select(ctx, &proofSetRootDeletes, ` + SELECT id, tx_hash, roots, set_id + FROM pdp_delete_root + WHERE tx_hash IS NOT NULL`) + if err != nil { + return xerrors.Errorf("failed to select proof set deletes: %w", err) + } + + if len(proofSetRootDeletes) == 0 { + return nil + } + + for _, psd := range proofSetRootDeletes { + err := processProofSetRootDelete(ctx, db, psd, ethClient) + if err != nil { + log.Warnf("Failed to process proof set root delete for tx %s: %v", psd.Hash, err) + continue + } + } + + return nil +} + +func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetRootDelete, ethClient *ethclient.Client) error { + var txReceiptJSON []byte + var txSuccess bool + err := db.QueryRow(ctx, ` + SELECT tx_success, tx_receipt + FROM message_waits_eth + WHERE signed_tx_hash = $1 + `, psd.Hash).Scan(&txReceiptJSON, &txSuccess) + if err != nil { + return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psd.Hash, err) + } + + var txReceipt types.Receipt + err = json.Unmarshal(txReceiptJSON, &txReceipt) + if err != nil { + return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", psd.Hash, err) + } + + if !txSuccess { + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`DELETE FROM pdp_delete_root WHERE id = $1`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete row from pdp_delete_root: %w", err) + } + return true, nil + }) + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + return nil + } + + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, + remove_deal_id = $1, + remove_message_hash = $2 + WHERE id = $3 AND root = ANY($4)`, psd.ID, psd.Hash, psd.ProofSet, psd.Roots) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + n, err = tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + return true, nil + }) + + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + return nil +} diff --git a/tasks/pdp/proofset_delete_watch.go b/tasks/pdp/proofset_delete_watch.go new file mode 100644 index 000000000..2dc070026 --- /dev/null +++ b/tasks/pdp/proofset_delete_watch.go @@ -0,0 +1,161 @@ +package pdp + +import ( + "context" + "encoding/json" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/chainsched" + + chainTypes "github.com/filecoin-project/lotus/chain/types" +) + +type ProofSetDelete struct { + DeleteMessageHash string `db:"tx_hash"` + ID string `db:"id"` + PID int64 `db:"set_id"` +} + +func NewWatcherDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { + if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { + err := processPendingProofSetDeletes(ctx, db, ethClient) + if err != nil { + log.Warnf("Failed to process pending proof set creates: %v", err) + } + return nil + }); err != nil { + panic(err) + } +} + +func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { + // Query for pdp_proof_set_delete where txHash is not NULL + var proofSetDeletes []ProofSetDelete + + err := db.Select(ctx, &proofSetDeletes, ` + SELECT id, client, tx_hash, + FROM pdp_proof_set_delete + WHERE tx_hash IS NOT NULL`) + if err != nil { + return xerrors.Errorf("failed to select proof set deletes: %w", err) + } + + if len(proofSetDeletes) == 0 { + // No pending proof set creates + return nil + } + + // Process each proof set delete + for _, psd := range proofSetDeletes { + err := processProofSetDelete(ctx, db, psd, ethClient) + if err != nil { + log.Warnf("Failed to process proof set delete for tx %s: %v", psd.DeleteMessageHash, err) + continue + } + } + + return nil +} + +func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDelete, ethClient *ethclient.Client) error { + // Retrieve the tx_receipt from message_waits_eth + var txReceiptJSON []byte + var txSuccess bool + err := db.QueryRow(ctx, ` + SELECT tx_success, tx_receipt + FROM message_waits_eth + WHERE signed_tx_hash = $1 + `, psd.DeleteMessageHash).Scan(&txReceiptJSON, &txSuccess) + if err != nil { + return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psd.DeleteMessageHash, err) + } + + // Unmarshal the tx_receipt JSON into types.Receipt + var txReceipt types.Receipt + err = json.Unmarshal(txReceiptJSON, &txReceipt) + if err != nil { + return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", psd.DeleteMessageHash, err) + } + + // Exit early if transaction executed with failure + if !txSuccess { + // This means msg failed, we should let the user know + // TODO: Review if error would be in receipt + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`DELETE FROM pdp_proof_set_delete WHERE id = $1`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete row from pdp_proof_set_delete: %w", err) + } + return true, nil + }) + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + return nil + } + + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_proof_set SET removed = TRUE, + remove_deal_id = $1, + remove_message_hash = $2 + WHERE id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + _, err = tx.Exec(`DELETE FROM pdp_proof_set_delete WHERE id = $1`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete row from pdp_proof_set_delete: %w", err) + } + n, err = tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + n, err = tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, + remove_deal_id = $1, + remove_message_hash = $2 + WHERE id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + return true, nil + }) + + if err != nil { + return xerrors.Errorf("failed to commit transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + + return nil +} diff --git a/tasks/pdp/task_delete_root.go b/tasks/pdp/task_delete_root.go new file mode 100644 index 000000000..f94abbf36 --- /dev/null +++ b/tasks/pdp/task_delete_root.go @@ -0,0 +1,182 @@ +package pdp + +import ( + "context" + "math/big" + "strings" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/pdp/contract" + "github.com/filecoin-project/curio/tasks/message" +) + +type PDPTaskDeleteRoot struct { + db *harmonydb.DB + sender *message.SenderETH + ethClient *ethclient.Client +} + +func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var rdeletes []struct { + ID string `db:"id"` + SetID int64 `db:"set_id"` + Roots []int64 `db:"roots"` + ExtraData []byte `db:"extra_data"` + } + + err = p.db.Select(ctx, &rdeletes, `SELECT id, set_id, roots, extra_data FROM pdp_delete_root WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to get task details from DB: %w", err) + } + + if len(rdeletes) != 1 { + return false, xerrors.Errorf("incorrect rows for delete root found for taskID %d", taskID) + } + + rdelete := rdeletes[0] + + extraDataBytes := []byte{} + + if rdelete.ExtraData != nil { + extraDataBytes = rdelete.ExtraData + } + + proofSetID := new(big.Int).SetUint64(uint64(rdelete.SetID)) + + pdpContracts := contract.ContractAddresses() + pdpVerifierAddress := pdpContracts.PDPVerifier + + pdpVerifier, err := contract.NewPDPVerifier(pdpVerifierAddress, p.ethClient) + if err != nil { + return false, xerrors.Errorf("failed to instantiate PDPVerifier contract at %s: %w", pdpVerifierAddress.Hex(), err) + } + + callOpts := &bind.CallOpts{ + Context: ctx, + } + + // Get the sender address for this proofset + owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + if err != nil { + return false, xerrors.Errorf("failed to get owner: %w", err) + } + + var roots []*big.Int + for _, root := range rdelete.Roots { + roots = append(roots, new(big.Int).SetUint64(uint64(root))) + } + + abiData, err := contract.PDPVerifierMetaData.GetAbi() + if err != nil { + return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) + } + + // Pack the method call data + data, err := abiData.Pack("scheduleRemovals", proofSetID, roots, extraDataBytes) + if err != nil { + return false, xerrors.Errorf("packing data: %w", err) + } + + // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) + tx := types.NewTransaction( + 0, + contract.ContractAddresses().PDPVerifier, + contract.SybilFee(), + 0, + nil, + data, + ) + + // Send the transaction using SenderETH + reason := "pdp-rmroot" + txHash, err := p.sender.Send(ctx, owner, tx, reason) + if err != nil { + return false, xerrors.Errorf("sending transaction: %w", err) + } + + // Insert into message_waits_eth and pdp_proof_set_delete + txHashLower := strings.ToLower(txHash.Hex()) + n, err := p.db.Exec(ctx, `UPDATE pdp_delete_root SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_delete_root: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_delete_root: %d", n) + } + + return true, nil +} + +func (p *PDPTaskDeleteRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (p *PDPTaskDeleteRoot) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "PDPDeleteRoot", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return p.schedule(context.Background(), taskFunc) + }), + } +} + +func (p *PDPTaskDeleteRoot) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_delete_root + WHERE task_id IS NULL + AND tx_hash IS NULL`).Scan(&did) + if err != nil { + return false, xerrors.Errorf("failed to query pdp_delete_root: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid deal ID found for scheduling") + } + + _, err = tx.Exec(`UPDATE pdp_delete_root SET task_id = $1, WHERE id = $2 AND task_id IS NULL AND tx_hash IS NULL`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_delete_root: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (p *PDPTaskDeleteRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} + +func NewPDPTaskDeleteRoot(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskDeleteRoot { + return &PDPTaskDeleteRoot{ + db: db, + sender: sender, + ethClient: ethClient, + } +} + +var _ harmonytask.TaskInterface = &PDPTaskDeleteRoot{} diff --git a/tasks/pdp/task_delete_rootset.go b/tasks/pdp/task_delete_rootset.go new file mode 100644 index 000000000..b9441b937 --- /dev/null +++ b/tasks/pdp/task_delete_rootset.go @@ -0,0 +1,175 @@ +package pdp + +import ( + "context" + "math/big" + "strings" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/pdp/contract" + "github.com/filecoin-project/curio/tasks/message" +) + +type PDPTaskDeleteProofSet struct { + db *harmonydb.DB + sender *message.SenderETH + ethClient *ethclient.Client + filClient PDPServiceNodeApi +} + +func NewPDPTaskDeleteProofSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskDeleteProofSet { + return &PDPTaskDeleteProofSet{ + db: db, + sender: sender, + ethClient: ethClient, + filClient: filClient, + } +} + +func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + var pdeletes []struct { + SetID int64 `db:"set_id"` + ExtraData []byte `db:"extra_data"` + } + + err = p.db.Select(ctx, &pdeletes, `SELECT set_id, extra_data FROM pdp_proof_set_create WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to get task details from DB: %w", err) + } + + if len(pdeletes) != 0 { + return false, xerrors.Errorf("incorrect rows for proofset delete found for taskID %d", taskID) + } + + pdelete := pdeletes[0] + + extraDataBytes := []byte{} + + proofSetID := new(big.Int).SetUint64(uint64(pdelete.SetID)) + + if pdelete.ExtraData != nil { + extraDataBytes = pdelete.ExtraData + } + + pdpContracts := contract.ContractAddresses() + pdpVerifierAddress := pdpContracts.PDPVerifier + + pdpVerifier, err := contract.NewPDPVerifier(pdpVerifierAddress, p.ethClient) + if err != nil { + return false, xerrors.Errorf("failed to instantiate PDPVerifier contract at %s: %w", pdpVerifierAddress.Hex(), err) + } + + callOpts := &bind.CallOpts{ + Context: ctx, + } + + // Get the sender address for this proofset + owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + if err != nil { + return false, xerrors.Errorf("failed to get owner: %w", err) + } + + // Manually create the transaction without requiring a Signer + // Obtain the ABI of the PDPVerifier contract + abiData, err := contract.PDPVerifierMetaData.GetAbi() + if err != nil { + return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) + } + + // Pack the method call data + data, err := abiData.Pack("deleteProofSet", proofSetID, extraDataBytes) + if err != nil { + return false, xerrors.Errorf("packing data: %w", err) + } + + // Prepare the transaction (nonce will be set to 0, SenderETH will assign it) + tx := types.NewTransaction( + 0, + contract.ContractAddresses().PDPVerifier, + contract.SybilFee(), + 0, + nil, + data, + ) + + // Send the transaction using SenderETH + reason := "pdp-rmproofset" + txHash, err := p.sender.Send(ctx, owner, tx, reason) + if err != nil { + return false, xerrors.Errorf("sending transaction: %w", err) + } + + // Insert into message_waits_eth and pdp_proof_set_delete + txHashLower := strings.ToLower(txHash.Hex()) + n, err := p.db.Exec(ctx, `UPDATE pdp_proof_set_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_delete: %d", n) + } + return true, nil +} + +func (p *PDPTaskDeleteProofSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (p *PDPTaskDeleteProofSet) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "PDPAddProofSet", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(3*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return p.schedule(context.Background(), taskFunc) + }), + } +} + +func (p *PDPTaskDeleteProofSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_proof_set_delete WHERE task_id IS NULL AND tx_hash IS NULL`).Scan(&id) + if err != nil { + return false, xerrors.Errorf("failed to query pdp_proof_set_delete: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid id found for taskID") + } + + _, err = tx.Exec(`UPDATE pdp_proof_set_delete SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (p *PDPTaskDeleteProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &PDPTaskDeleteProofSet{} diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go index 445dafb36..4ac68fa3f 100644 --- a/tasks/pdp/task_save_cache.go +++ b/tasks/pdp/task_save_cache.go @@ -569,11 +569,6 @@ func NewCommPWithSize(size uint64) *Calc { return c } -const ( - targetReadSize = 4 * 1024 * 1024 // 4 MiB - inputBytesPerLeaf = 127 // raw input bytes that become one 32-byte leaf -) - func (cp *Calc) snapshotLayerIndex(size uint64, test bool) { if size == 0 { panic("size must be > 0") @@ -591,8 +586,8 @@ func (cp *Calc) snapshotLayerIndex(size uint64, test bool) { //Calculate layer L such that 127 * 2^L >= targetReadSize //ā†’ 2^L >= targetReadSize / 32 //ratio := float64(1040384) / 32 - testRatio := float64(2032) / 32 - ProdRatio := float64(4161536) / 32 + testRatio := float64(2032) / LeafSize // 2 KiB.UnPadded() + ProdRatio := float64(4161536) / LeafSize // 4 MiB.UnPadded() var layer int if test { layer = int(math.Ceil(math.Log2(testRatio))) From f3b430d65e84e749e73f376c420919d99351955f Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 22 Jul 2025 19:10:04 +0400 Subject: [PATCH 22/55] serial upload --- harmony/harmonydb/sql/20240228-piece-park.sql | 2 + .../harmonydb/sql/20250505-market_mk20.sql | 44 +- lib/ffi/piece_funcs.go | 24 +- market/mk20/http/docs.go | 169 ++++++- market/mk20/http/http.go | 135 +++++- market/mk20/http/swagger.json | 169 ++++++- market/mk20/http/swagger.yaml | 129 ++++- market/mk20/mk20.go | 161 +++++-- market/mk20/mk20_upload.go | 448 +++++++++++++++++- market/mk20/mk20_utils.go | 25 +- market/mk20/utils.go | 13 +- tasks/pdp/task_aggregation.go | 37 +- tasks/piece/task_aggregate_chunks.go | 43 +- tasks/storage-market/mk20.go | 1 - tasks/storage-market/task_aggregation.go | 37 +- 15 files changed, 1305 insertions(+), 132 deletions(-) diff --git a/harmony/harmonydb/sql/20240228-piece-park.sql b/harmony/harmonydb/sql/20240228-piece-park.sql index b4fbfeffa..0b8903bc7 100644 --- a/harmony/harmonydb/sql/20240228-piece-park.sql +++ b/harmony/harmonydb/sql/20240228-piece-park.sql @@ -13,6 +13,8 @@ create table parked_pieces ( -- long_term boolean not null default false, -- Added in 20240930-pdp.sql + -- skip boolean not null default false, -- Added in 20250505-market_mk20.sql to allow skipping download + -- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql foreign key (task_id) references harmony_task (id) on delete set null, -- dropped foreign key (cleanup_task_id) references harmony_task (id) on delete set null, -- dropped diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index cbc0a1951..fa8214ffc 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -34,6 +34,10 @@ ALTER TABLE market_piece_deal ALTER TABLE market_piece_deal ADD COLUMN piece_ref BIGINT; +-- Add column to skip scheduling piece_park. Used for upload pieces +ALTER TABLE parked_pieces + ADD COLUMN skip BOOLEAN NOT NULL DEFAULT FALSE; + -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). CREATE OR REPLACE FUNCTION process_piece_deal( @@ -158,6 +162,8 @@ CREATE TABLE ddo_contracts ( abi TEXT NOT NULL ); +-- This is main MK20 Deal table. Rows are added per deal and some +-- modification is allowed later CREATE TABLE market_mk20_deal ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), id TEXT PRIMARY KEY, @@ -174,6 +180,7 @@ CREATE TABLE market_mk20_deal ( pdp_v1 JSONB NOT NULL DEFAULT 'null' ); +-- This is main pipeline table for PoRep processing of MK20 deals CREATE TABLE market_mk20_pipeline ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), id TEXT NOT NULL, @@ -219,14 +226,25 @@ CREATE TABLE market_mk20_pipeline ( PRIMARY KEY (id, aggr_index) ); +-- This table is used to hold MK20 deals waiting for PoRep pipeline +-- to process. This allows disconnecting the need to immediately process +-- deals as received and allow upload later strategy to work CREATE TABLE market_mk20_pipeline_waiting ( id TEXT PRIMARY KEY ); +-- This table is used to keep track of deals which need data upload. +-- A separate table helps easier status check, chunked+serial upload support CREATE TABLE market_mk20_upload_waiting ( - id TEXT PRIMARY KEY + id TEXT PRIMARY KEY, + chunked BOOLEAN DEFAULT NULL, + ref_id BIGINT DEFAULT NULL ); +-- This table help disconnected downloads from main PoRep/PDP pipelines +-- It helps with allowing multiple downloads per deal i.e. server side aggregation. +-- This also allows us to reuse ongoing downloads within the same deal aggregation. +-- It also allows using a common download pipeline for both PoRep and PDP. CREATE TABLE market_mk20_download_pipeline ( id TEXT NOT NULL, product TEXT NOT NULL, -- This allows us to run multiple refs per product for easier lifecycle management @@ -236,6 +254,7 @@ CREATE TABLE market_mk20_download_pipeline ( PRIMARY KEY (id, product, piece_cid, piece_size) ); +-- Offline URLs for PoRep deals. CREATE TABLE market_mk20_offline_urls ( id TEXT NOT NULL, piece_cid TEXT NOT NULL, @@ -246,6 +265,8 @@ CREATE TABLE market_mk20_offline_urls ( PRIMARY KEY (id, piece_cid, piece_size) ); +-- This table tracks the chunk upload progress for a MK20 deal. Common for both +-- PoRep and PDP CREATE TABLE market_mk20_deal_chunk ( id TEXT not null, chunk INT not null, @@ -257,16 +278,19 @@ CREATE TABLE market_mk20_deal_chunk ( PRIMARY KEY (id, chunk) ); +-- MK20 product and their status table CREATE TABLE market_mk20_products ( name TEXT PRIMARY KEY, enabled BOOLEAN DEFAULT TRUE ); +-- MK20 supported data sources and their status table CREATE TABLE market_mk20_data_source ( name TEXT PRIMARY KEY, enabled BOOLEAN DEFAULT TRUE ); +-- Add products and data sources to table INSERT INTO market_mk20_products (name, enabled) VALUES ('ddo_v1', TRUE); INSERT INTO market_mk20_products (name, enabled) VALUES ('retrieval_v1', TRUE); INSERT INTO market_mk20_products (name, enabled) VALUES ('pdp_v1', TRUE); @@ -275,6 +299,9 @@ INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('put', TRUE); +-- This function triggers a download for an offline piece. +-- It is different from MK1.2 PoRep pipeline as it download the offline pieces +-- locally. This is to allow serving retrievals with piece park. CREATE OR REPLACE FUNCTION process_offline_download( _id TEXT, _piece_cid TEXT, @@ -344,10 +371,7 @@ BEGIN END; $$ LANGUAGE plpgsql; --- Add column to skip scheduling piece_park -ALTER TABLE parked_pieces - ADD COLUMN skip BOOLEAN DEFAULT FALSE; - +-- Main ProofSet table for PDP CREATE TABLE pdp_proof_set ( id BIGINT PRIMARY KEY, -- on-chain proofset id client TEXT NOT NULL, -- client wallet which requested this proofset @@ -387,6 +411,7 @@ CREATE TABLE pdp_proof_set ( unique (remove_deal_id) ); +-- ProofSet create table governs the PoofSet create task CREATE TABLE pdp_proof_set_create ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -398,6 +423,7 @@ CREATE TABLE pdp_proof_set_create ( tx_hash TEXT DEFAULT NULL ); +-- ProofSet delete table governs the PoofSet delete task CREATE TABLE pdp_proof_set_delete ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -409,6 +435,7 @@ CREATE TABLE pdp_proof_set_delete ( tx_hash TEXT DEFAULT NULL ); +-- This table governs the delete root tasks CREATE TABLE pdp_delete_root ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -421,6 +448,7 @@ CREATE TABLE pdp_delete_root ( tx_hash TEXT DEFAULT NULL ); +-- Main ProofSet Root table. Any and all root ever added by SP must be part of this table CREATE TABLE pdp_proofset_root ( proofset BIGINT NOT NULL, -- pdp_proof_sets.id client TEXT NOT NULL, @@ -449,7 +477,7 @@ CREATE TABLE pdp_proofset_root ( CREATE TABLE pdp_pipeline ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), - id TEXT PRIMARY KEY, + id TEXT NOT NULL, client TEXT NOT NULL, piece_cid_v2 TEXT NOT NULL, -- v2 piece_cid @@ -486,7 +514,9 @@ CREATE TABLE pdp_pipeline ( indexing_task_id BIGINT DEFAULT NULL, indexed BOOLEAN DEFAULT FALSE, - complete BOOLEAN DEFAULT FALSE + complete BOOLEAN DEFAULT FALSE, + + PRIMARY KEY (id, aggr_index) ); CREATE TABLE market_mk20_clients ( diff --git a/lib/ffi/piece_funcs.go b/lib/ffi/piece_funcs.go index b01625c6a..729432e93 100644 --- a/lib/ffi/piece_funcs.go +++ b/lib/ffi/piece_funcs.go @@ -78,11 +78,11 @@ func (sb *SealCalls) RemovePiece(ctx context.Context, id storiface.PieceNumber) return sb.sectors.storage.Remove(ctx, id.Ref().ID, storiface.FTPiece, true, nil) } -func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType) (abi.PieceInfo, error) { +func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType, verifySize bool) (abi.PieceInfo, uint64, error) { // Use storageType in AcquireSector paths, _, done, err := sb.sectors.AcquireSector(ctx, nil, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType) if err != nil { - return abi.PieceInfo{}, err + return abi.PieceInfo{}, 0, err } defer done() @@ -91,7 +91,7 @@ func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.Pie destFile, err := os.OpenFile(tempDest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - return abi.PieceInfo{}, xerrors.Errorf("creating temp piece file '%s': %w", tempDest, err) + return abi.PieceInfo{}, 0, xerrors.Errorf("creating temp piece file '%s': %w", tempDest, err) } removeTemp := true @@ -112,36 +112,36 @@ func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.Pie n, err := io.CopyBuffer(writers, io.LimitReader(data, size), make([]byte, 8<<20)) if err != nil { _ = destFile.Close() - return abi.PieceInfo{}, xerrors.Errorf("copying piece data: %w", err) + return abi.PieceInfo{}, 0, xerrors.Errorf("copying piece data: %w", err) } if err := destFile.Close(); err != nil { - return abi.PieceInfo{}, xerrors.Errorf("closing temp piece file: %w", err) + return abi.PieceInfo{}, 0, xerrors.Errorf("closing temp piece file: %w", err) } - if n != size { - return abi.PieceInfo{}, xerrors.Errorf("short write: %d", n) + if verifySize && n != size { + return abi.PieceInfo{}, 0, xerrors.Errorf("short write: %d", n) } digest, pieceSize, err := wr.Digest() if err != nil { - return abi.PieceInfo{}, xerrors.Errorf("computing piece digest: %w", err) + return abi.PieceInfo{}, 0, xerrors.Errorf("computing piece digest: %w", err) } pcid, err := commcid.DataCommitmentV1ToCID(digest) if err != nil { - return abi.PieceInfo{}, xerrors.Errorf("computing piece CID: %w", err) + return abi.PieceInfo{}, 0, xerrors.Errorf("computing piece CID: %w", err) } psize := abi.PaddedPieceSize(pieceSize) copyEnd := time.Now() - log.Infow("wrote piece", "piece", pieceID, "size", size, "duration", copyEnd.Sub(copyStart), "dest", dest, "MiB/s", float64(size)/(1<<20)/copyEnd.Sub(copyStart).Seconds()) + log.Infow("wrote piece", "piece", pieceID, "size", n, "duration", copyEnd.Sub(copyStart), "dest", dest, "MiB/s", float64(size)/(1<<20)/copyEnd.Sub(copyStart).Seconds()) if err := os.Rename(tempDest, dest); err != nil { - return abi.PieceInfo{}, xerrors.Errorf("rename temp piece to dest %s -> %s: %w", tempDest, dest, err) + return abi.PieceInfo{}, 0, xerrors.Errorf("rename temp piece to dest %s -> %s: %w", tempDest, dest, err) } removeTemp = false - return abi.PieceInfo{PieceCID: pcid, Size: psize}, nil + return abi.PieceInfo{PieceCID: pcid, Size: psize}, uint64(n), nil } diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 9f4d5646a..7d3fd6955 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -329,7 +329,170 @@ const docTemplate = `{ } } }, - "/upload/finalize/{id}": { + "/upload/{id}": { + "put": { + "description": "Allows uploading data for deals in a single stream. Suitable for small deals.", + "summary": "Upload the deal data", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "raw binary", + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } + } + ], + "responses": { + "200": { + "description": "UploadOk indicates a successful upload operation, represented by the HTTP status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "500": { + "description": "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + } + } + }, + "post": { + "description": "Finalizes the serial upload process once data has been uploaded", + "consumes": [ + "application/json" + ], + "summary": "Finalizes the serial upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.deal in json format", + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/uploads/finalize/{id}": { "post": { "description": "Finalizes the upload process once all the chunks are uploaded.", "consumes": [ @@ -441,7 +604,7 @@ const docTemplate = `{ } } }, - "/upload/{id}": { + "/uploads/{id}": { "get": { "description": "Return a json struct detailing the current status of a deal upload.", "summary": "Status of deal upload", @@ -533,7 +696,7 @@ const docTemplate = `{ } } }, - "/upload/{id}/{chunkNum}": { + "/uploads/{id}/{chunkNum}": { "put": { "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", "summary": "Upload a file chunk", diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 4ea0413a7..0dd16e416 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -134,13 +134,15 @@ func APIRouter(mdh *MK20DealHandler, domainName string) http.Handler { mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "request timeout")) mux.Method("GET", "/status/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "request timeout")) mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) - mux.Method("POST", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStart), requestTimeout, "request timeout")) - mux.Method("GET", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStatus), requestTimeout, "request timeout")) - mux.Put("/upload/{id}/{chunkNum}", mdh.mk20UploadDealChunks) - mux.Method("POST", "/upload/finalize/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20FinalizeUpload), requestTimeout, "request timeout")) + mux.Method("POST", "/uploads/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStart), requestTimeout, "request timeout")) + mux.Method("GET", "/uploads/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStatus), requestTimeout, "request timeout")) + mux.Put("/uploads/{id}/{chunkNum}", mdh.mk20UploadDealChunks) + mux.Method("POST", "/uploads/finalize/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20FinalizeUpload), requestTimeout, "request timeout")) mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) mux.Method("POST", "/update/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UpdateDeal), requestTimeout, "request timeout")) + mux.Method("POST", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20SerialUploadFinalize), requestTimeout, "request timeout")) + mux.Put("/upload/{id}", mdh.mk20SerialUpload) return mux } @@ -385,7 +387,7 @@ func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http. } // mk20UploadStatus handles the upload status requests for a given id. -// @Router /upload/{id} [get] +// @Router /uploads/{id} [get] // @Param id path string true "id" // @Summary Status of deal upload // @Description Return a json struct detailing the current status of a deal upload. @@ -412,7 +414,7 @@ func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Requ } // mk20UploadDealChunks handles uploading of deal file chunks. -// @Router /upload/{id}/{chunkNum} [put] +// @Router /uploads/{id}/{chunkNum} [put] // @Summary Upload a file chunk // @Description Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads. // @BasePath /market/mk20 @@ -465,7 +467,7 @@ func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http. } // mk20UploadStart handles the initiation of an upload process for MK20 deal data. -// @Router /upload/{id} [post] +// @Router /uploads/{id} [post] // @Summary Starts the upload process // @Description Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal. // @BasePath /market/mk20 @@ -519,7 +521,7 @@ func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Reque } // mk20FinalizeUpload finalizes the upload process for a given deal by processing the request and updating the associated deal in the system if required. -// @Router /upload/finalize/{id} [post] +// @Router /uploads/finalize/{id} [post] // @Summary Finalizes the upload process // @Description Finalizes the upload process once all the chunks are uploaded. // @BasePath /market/mk20 @@ -655,5 +657,120 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques log.Infow("received deal update proposal", "body", string(body)) - mdh.dm.MK20Handler.UpdateDeal(id, &deal, w) + result := mdh.dm.MK20Handler.UpdateDeal(id, &deal) + + log.Infow("deal updated", + "id", deal.Identifier, + "HTTPCode", result.HTTPCode, + "Reason", result.Reason) + + w.WriteHeader(int(result.HTTPCode)) + _, err = w.Write([]byte(fmt.Sprint("Reason: ", result.Reason))) + if err != nil { + log.Errorw("writing deal update response:", "id", deal.Identifier, "error", err) + } +} + +// mk20SerialUpload handles uploading of deal data in a single stream +// @Router /upload/{id} [put] +// @Summary Upload the deal data +// @Description Allows uploading data for deals in a single stream. Suitable for small deals. +// @BasePath /market/mk20 +// @Param id path string true "id" +// @accepts bytes +// @Param body body []byte true "raw binary" +// @Failure 200 {object} mk20.UploadCode "UploadOk indicates a successful upload operation, represented by the HTTP status code 200" +// @Failure 500 {object} mk20.UploadCode "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500" +// @Failure 404 {object} mk20.UploadStartCode "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" +func (mdh *MK20DealHandler) mk20SerialUpload(w http.ResponseWriter, r *http.Request) { + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + mdh.dm.MK20Handler.HandleSerialUpload(id, r.Body, w) +} + +// mk20SerialUploadFinalize finalizes the serial upload process for a given deal by processing the request and updating the associated deal in the system if required. +// @Router /upload/{id} [post] +// @Summary Finalizes the serial upload process +// @Description Finalizes the serial upload process once data has been uploaded +// @BasePath /market/mk20 +// @Param id path string true "id" +// @accepts json +// @Param body body mk20.Deal optional "mk20.deal in json format" +// @Accept json +// @Failure 200 {object} mk20.DealCode "Ok represents a successful operation with an HTTP status code of 200" +// @Failure 400 {object} mk20.DealCode "ErrBadProposal represents a validation error that indicates an invalid or malformed proposal input in the context of validation logic" +// @Failure 404 {object} mk20.DealCode "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404" +// @Failure 430 {object} mk20.DealCode "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data" +// @Failure 422 {object} mk20.DealCode "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context" +// @Failure 423 {object} mk20.DealCode "ErrUnsupportedProduct indicates that the requested product is not supported by the provider" +// @Failure 424 {object} mk20.DealCode "ErrProductNotEnabled indicates that the requested product is not enabled on the provider" +// @Failure 425 {object} mk20.DealCode "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data" +// @Failure 426 {object} mk20.DealCode "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules" +// @Failure 500 {object} mk20.DealCode "ErrServerInternalError indicates an internal server error with a corresponding error code of 500" +// @Failure 503 {object} mk20.DealCode "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503" +// @Failure 429 {object} mk20.DealCode "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment" +// @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" +// @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" +// @Failure 400 {string} string "Bad Request - Invalid input or validation error" +func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *http.Request) { + idStr := chi.URLParam(r, "id") + if idStr == "" { + log.Errorw("missing id in url", "url", r.URL) + http.Error(w, "missing id in url", http.StatusBadRequest) + return + } + + id, err := ulid.Parse(idStr) + if err != nil { + log.Errorw("invalid id in url", "id", idStr, "err", err) + http.Error(w, "invalid id in url", http.StatusBadRequest) + return + } + + ct := r.Header.Get("Content-Type") + // If Content-Type is not set this is does not require updating the deal + if len(ct) == 0 { + log.Infow("received finalize upload proposal without content type", "id", id) + mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, nil, w) + return + } + + var deal mk20.Deal + if ct != "application/json" { + log.Errorf("invalid content type: %s", ct) + http.Error(w, "invalid content type", http.StatusBadRequest) + return + } + + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + log.Errorf("error reading request body: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err = json.Unmarshal(body, &deal) + if err != nil { + log.Errorf("error unmarshaling json: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + log.Infow("received serial upload finalize proposal", "body", string(body)) + + mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, &deal, w) } diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 3a7d62b70..a8740391d 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -320,7 +320,170 @@ } } }, - "/upload/finalize/{id}": { + "/upload/{id}": { + "put": { + "description": "Allows uploading data for deals in a single stream. Suitable for small deals.", + "summary": "Upload the deal data", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "raw binary", + "name": "body", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "integer" + } + } + } + ], + "responses": { + "200": { + "description": "UploadOk indicates a successful upload operation, represented by the HTTP status code 200", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process", + "schema": { + "$ref": "#/definitions/mk20.UploadStartCode" + } + }, + "500": { + "description": "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500", + "schema": { + "$ref": "#/definitions/mk20.UploadCode" + } + } + } + }, + "post": { + "description": "Finalizes the serial upload process once data has been uploaded", + "consumes": [ + "application/json" + ], + "summary": "Finalizes the serial upload process", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + }, + { + "description": "mk20.deal in json format", + "name": "body", + "in": "body", + "schema": { + "$ref": "#/definitions/mk20.Deal" + } + } + ], + "responses": { + "200": { + "description": "Ok represents a successful operation with an HTTP status code of 200", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "400": { + "description": "Bad Request - Invalid input or validation error", + "schema": { + "type": "string" + } + }, + "404": { + "description": "ErrDealNotFound indicates that the specified deal could not be found, corresponding to the HTTP status code 404", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "422": { + "description": "ErrUnsupportedDataSource indicates the specified data source is not supported or disabled for use in the current context", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "423": { + "description": "ErrUnsupportedProduct indicates that the requested product is not supported by the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "424": { + "description": "ErrProductNotEnabled indicates that the requested product is not enabled on the provider", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "425": { + "description": "ErrProductValidationFailed indicates a failure during product-specific validation due to invalid or missing data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "426": { + "description": "ErrDealRejectedByMarket indicates that a proposed deal was rejected by the market for not meeting its acceptance criteria or rules", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "429": { + "description": "ErrServiceOverloaded indicates that the service is overloaded and cannot process the request at the moment", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "430": { + "description": "ErrMalformedDataSource indicates that the provided data source is incorrectly formatted or contains invalid data", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "440": { + "description": "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "441": { + "description": "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "500": { + "description": "ErrServerInternalError indicates an internal server error with a corresponding error code of 500", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + }, + "503": { + "description": "ErrServiceMaintenance indicates that the service is temporarily unavailable due to maintenance, corresponding to HTTP status code 503", + "schema": { + "$ref": "#/definitions/mk20.DealCode" + } + } + } + } + }, + "/uploads/finalize/{id}": { "post": { "description": "Finalizes the upload process once all the chunks are uploaded.", "consumes": [ @@ -432,7 +595,7 @@ } } }, - "/upload/{id}": { + "/uploads/{id}": { "get": { "description": "Return a json struct detailing the current status of a deal upload.", "summary": "Status of deal upload", @@ -524,7 +687,7 @@ } } }, - "/upload/{id}/{chunkNum}": { + "/uploads/{id}/{chunkNum}": { "put": { "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", "summary": "Upload a file chunk", diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index 47e98df71..dfb732bc8 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -584,6 +584,131 @@ paths: $ref: '#/definitions/mk20.DealCode' summary: Update the deal details of existing deals. /upload/{id}: + post: + consumes: + - application/json + description: Finalizes the serial upload process once data has been uploaded + parameters: + - description: id + in: path + name: id + required: true + type: string + - description: mk20.deal in json format + in: body + name: body + schema: + $ref: '#/definitions/mk20.Deal' + responses: + "200": + description: Ok represents a successful operation with an HTTP status code + of 200 + schema: + $ref: '#/definitions/mk20.DealCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: ErrDealNotFound indicates that the specified deal could not + be found, corresponding to the HTTP status code 404 + schema: + $ref: '#/definitions/mk20.DealCode' + "422": + description: ErrUnsupportedDataSource indicates the specified data source + is not supported or disabled for use in the current context + schema: + $ref: '#/definitions/mk20.DealCode' + "423": + description: ErrUnsupportedProduct indicates that the requested product + is not supported by the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "424": + description: ErrProductNotEnabled indicates that the requested product is + not enabled on the provider + schema: + $ref: '#/definitions/mk20.DealCode' + "425": + description: ErrProductValidationFailed indicates a failure during product-specific + validation due to invalid or missing data + schema: + $ref: '#/definitions/mk20.DealCode' + "426": + description: ErrDealRejectedByMarket indicates that a proposed deal was + rejected by the market for not meeting its acceptance criteria or rules + schema: + $ref: '#/definitions/mk20.DealCode' + "429": + description: ErrServiceOverloaded indicates that the service is overloaded + and cannot process the request at the moment + schema: + $ref: '#/definitions/mk20.DealCode' + "430": + description: ErrMalformedDataSource indicates that the provided data source + is incorrectly formatted or contains invalid data + schema: + $ref: '#/definitions/mk20.DealCode' + "440": + description: ErrMarketNotEnabled indicates that the market is not enabled + for the requested operation + schema: + $ref: '#/definitions/mk20.DealCode' + "441": + description: ErrDurationTooShort indicates that the provided duration value + does not meet the minimum required threshold + schema: + $ref: '#/definitions/mk20.DealCode' + "500": + description: ErrServerInternalError indicates an internal server error with + a corresponding error code of 500 + schema: + $ref: '#/definitions/mk20.DealCode' + "503": + description: ErrServiceMaintenance indicates that the service is temporarily + unavailable due to maintenance, corresponding to HTTP status code 503 + schema: + $ref: '#/definitions/mk20.DealCode' + summary: Finalizes the serial upload process + put: + description: Allows uploading data for deals in a single stream. Suitable for + small deals. + parameters: + - description: id + in: path + name: id + required: true + type: string + - description: raw binary + in: body + name: body + required: true + schema: + items: + type: integer + type: array + responses: + "200": + description: UploadOk indicates a successful upload operation, represented + by the HTTP status code 200 + schema: + $ref: '#/definitions/mk20.UploadCode' + "400": + description: Bad Request - Invalid input or validation error + schema: + type: string + "404": + description: UploadStartCodeDealNotFound represents a 404 status indicating + the deal was not found during the upload start process + schema: + $ref: '#/definitions/mk20.UploadStartCode' + "500": + description: UploadServerError indicates a server-side error occurred during + the upload process, represented by the HTTP status code 500 + schema: + $ref: '#/definitions/mk20.UploadCode' + summary: Upload the deal data + /uploads/{id}: get: description: Return a json struct detailing the current status of a deal upload. parameters: @@ -653,7 +778,7 @@ paths: schema: $ref: '#/definitions/mk20.UploadStartCode' summary: Starts the upload process - /upload/{id}/{chunkNum}: + /uploads/{id}/{chunkNum}: put: description: Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads. @@ -702,7 +827,7 @@ paths: schema: $ref: '#/definitions/mk20.UploadCode' summary: Upload a file chunk - /upload/finalize/{id}: + /uploads/finalize/{id}: post: consumes: - application/json diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 614486cc0..4dbe232b9 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -134,7 +134,7 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti if deal.Products.DDOV1 != nil { // TODO: Remove this check once DDO market is done if build.BuildType == build.Build2k || build.BuildType == build.BuildDebug { - return m.processDDODeal(ctx, deal) + return m.processDDODeal(ctx, deal, nil) } log.Errorw("DDOV1 is not supported yet", "deal", deal.Identifier.String()) return &ProviderDealRejectionInfo{ @@ -146,7 +146,7 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti return m.processPDPDeal(ctx, deal) } -func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { +func (m *MK20) processDDODeal(ctx context.Context, deal *Deal, tx *harmonydb.Tx) *ProviderDealRejectionInfo { rejection, err := m.sanitizeDDODeal(ctx, deal) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) @@ -177,49 +177,75 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal) *ProviderDealReje // TODO: Backpressure, client filter - comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + process := func(tx *harmonydb.Tx) error { err = deal.SaveToDB(tx) if err != nil { - return false, err + return err } n, err := tx.Exec(`UPDATE market_mk20_deal SET ddo_v1 = jsonb_set(ddo_v1, '{deal_id}', to_jsonb($1::text)) WHERE id = $2;`, id, deal.Identifier.String()) if err != nil { - return false, err + return err } if n != 1 { - return false, fmt.Errorf("expected 1 row to be updated, got %d", n) + return fmt.Errorf("expected 1 row to be updated, got %d", n) } - if deal.Data.SourceHttpPut == nil { - _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + // Assume upload if no data source defined + if deal.Data == nil { + _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + } else { + if deal.Data.SourceHttpPut != nil { + _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + } else { + // All deals which are not upload should be entered in market_mk20_pipeline_waiting for further processing. + _, err = tx.Exec(`INSERT INTO market_mk20_pipeline_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + } } if err != nil { - return false, xerrors.Errorf("adding deal to waiting pipeline: %w", err) + return xerrors.Errorf("adding deal to waiting pipeline: %w", err) } - return true, nil - }) + return nil + } - if err != nil { - log.Errorw("error inserting deal into DB", "deal", deal, "error", err) - return &ProviderDealRejectionInfo{ - HTTPCode: ErrServerInternalError, + if tx != nil { + err := process(tx) + if err != nil { + log.Errorw("error inserting deal into DB", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + } } - } + } else { + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = process(tx) + if err != nil { + return false, err + } + return true, nil + }) - if !comm { - log.Errorw("error committing deal into DB", "deal", deal) - return &ProviderDealRejectionInfo{ - HTTPCode: ErrServerInternalError, + if err != nil { + log.Errorw("error inserting deal into DB", "deal", deal, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + } + } + + if !comm { + log.Errorw("error committing deal into DB", "deal", deal) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + } } } log.Debugw("deal inserted in DB", "deal", deal.Identifier.String()) return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusOK, + HTTPCode: Ok, } } @@ -364,12 +390,24 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje // If we have data source other that PUT then start the pipeline if deal.Data != nil { - if deal.Data.SourceHttpPut != nil || deal.Data.SourceAggregate != nil { + if deal.Data.SourceHTTP != nil || deal.Data.SourceAggregate != nil { err = insertPDPPipeline(ctx, tx, deal) if err != nil { return false, xerrors.Errorf("inserting pipeline: %w", err) } } + if deal.Data.SourceHttpPut != nil { + _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + if err != nil { + return false, xerrors.Errorf("inserting upload waiting: %w", err) + } + } + } else { + // Assume upload + _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + if err != nil { + return false, xerrors.Errorf("inserting upload waiting: %w", err) + } } pdp := deal.Products.PDPV1 @@ -708,15 +746,18 @@ func markDownloaded(ctx context.Context, db *harmonydb.DB) { } } -// UpdateDeal updates the details of a deal specified by its ID and writes the result or error to the provided HTTP response writer. +// UpdateDeal updates the details of a deal specified by its ID and returns ProviderDealRejectionInfo which has ErrorCode and Reason // @param id ulid.ULID // @param deal *Deal // @Return DealCode +// @Return Reason string -func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, w http.ResponseWriter) { +func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal) *ProviderDealRejectionInfo { if deal == nil { - http.Error(w, "deal not defined", int(ErrBadProposal)) - return + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "deal is undefined", + } } ctx := context.Background() @@ -728,34 +769,78 @@ func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, w http.ResponseWriter) { WHERE id = $1)`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if deal exists", "deal", id, "error", err) - http.Error(w, "", int(ErrServerInternalError)) - return + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", + } } if !exists { - http.Error(w, "", int(ErrDealNotFound)) - return + return &ProviderDealRejectionInfo{ + HTTPCode: ErrDealNotFound, + Reason: "", + } } code, nd, np, err := m.updateDealDetails(id, deal) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) if code == ErrServerInternalError { - http.Error(w, "", int(ErrServerInternalError)) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", + } } else { - http.Error(w, err.Error(), int(code)) + return &ProviderDealRejectionInfo{ + HTTPCode: code, + Reason: err.Error(), + } + } + } + + var rejection *ProviderDealRejectionInfo + + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // Save the updated deal to DB + err = nd.UpdateDeal(tx) + if err != nil { + return false, xerrors.Errorf("failed to update deal: %w", err) + } + + // Initiate new pipelines for DDO if required + for _, p := range np { + if p == ProductNameDDOV1 { + rejection = m.processDDODeal(ctx, nd, tx) + if rejection.HTTPCode != Ok { + return false, xerrors.Errorf("failed to process DDO deal") + } + } + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorw("failed to update deal details", "deal", id, "error", err) + if rejection != nil { + return rejection + } + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", } - return } - // Initiate new pipelines for DDO if required - for _, p := range np { - if p == ProductNameDDOV1 { - m.processDDODeal(ctx, nd) + if !comm { + log.Errorw("failed to commit deal details", "deal", id, "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", } } - w.WriteHeader(http.StatusOK) + return &ProviderDealRejectionInfo{ + HTTPCode: Ok, + Reason: "", + } } // To be used later for when data source is minerID diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index e812f5418..cab3619c2 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -17,6 +17,7 @@ import ( "golang.org/x/xerrors" commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/storiface" @@ -29,9 +30,11 @@ import ( func (m *MK20) HandleUploadStatus(ctx context.Context, id ulid.ULID, w http.ResponseWriter) { var exists bool err := m.DB.QueryRow(ctx, `SELECT EXISTS ( - SELECT 1 - FROM market_mk20_upload_waiting - WHERE id = $1;)`, id.String()).Scan(&exists) + SELECT 1 + FROM market_mk20_upload_waiting + WHERE id = $1 + AND (chunked IS NULL OR chunked = TRUE) + );`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if upload is waiting for data", "deal", id, "error", err) w.WriteHeader(int(UploadStatusCodeServerError)) @@ -127,8 +130,7 @@ func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, upload Start err := m.DB.QueryRow(ctx, `SELECT EXISTS ( SELECT 1 FROM market_mk20_upload_waiting - WHERE id = $1 - );`, id.String()).Scan(&exists) + WHERE id = $1 AND chunked IS NULL);`, id.String()).Scan(&exists) if err != nil { log.Errorw("failed to check if deal is waiting for upload to start", "deal", id, "error", err) http.Error(w, "", int(UploadStartCodeServerError)) @@ -210,6 +212,13 @@ func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, upload Start return false, xerrors.Errorf("closing insert chunk batch: %w", err) } } + n, err := tx.Exec(`UPDATE market_mk20_upload_waiting SET chunked = TRUE WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("updating chunked flag: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating chunked flag: expected 1 row updated, got %d", n) + } return true, nil }, harmonydb.OptionRetry()) if err != nil { @@ -284,7 +293,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w log.Debugw("uploading chunk", "deal", id, "chunk", chunk) chunkSize := chunkDetails[0].Size - reader := NewTimeoutReader(data, time.Second*5) + reader := NewTimeoutLimitReader(data, time.Second*5) m.maxParallelUploads.Add(1) // Generate unique tmp pieceCID and Size for parked_pieces tables @@ -360,7 +369,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w }() // Store the piece and generate PieceCID and Size - pi, err := m.sc.WriteUploadPiece(ctx, storiface.PieceNumber(pnum), chunkSize, reader, storiface.PathSealing) + pi, _, err := m.sc.WriteUploadPiece(ctx, storiface.PieceNumber(pnum), chunkSize, reader, storiface.PathSealing, true) if err != nil { log.Errorw("failed to write piece", "deal", id, "chunk", chunk, "error", err) http.Error(w, "", int(UploadServerError)) @@ -454,6 +463,10 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri return } + var rawSize uint64 + var newDeal *Deal + var dealUpdated bool + if deal != nil { // This is a deal where DataSource was not set - we should update the deal code, ndeal, _, err := m.updateDealDetails(id, deal) @@ -466,45 +479,80 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri } return } - rawSize, err := ndeal.RawSize() + rawSize, err = ndeal.RawSize() + if err != nil { + log.Errorw("failed to get raw size of deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + newDeal = ndeal + dealUpdated = true + } else { + rawSize, err = ddeal.RawSize() if err != nil { log.Errorw("failed to get raw size of deal", "deal", id, "error", err) http.Error(w, "", int(ErrServerInternalError)) return } + } - var valid bool + var valid bool - err = m.DB.QueryRow(ctx, `SELECT SUM(chunk_size) = $2 AS valid + err = m.DB.QueryRow(ctx, `SELECT SUM(chunk_size) = $2 AS valid FROM market_mk20_deal_chunk WHERE id = $1;`, id.String(), rawSize).Scan(&valid) + if err != nil { + log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if !valid { + log.Errorw("deal upload finalize failed", "deal", id, "error", "deal raw size does not match the sum of chunks") + http.Error(w, "deal raw size does not match the sum of chunks", int(ErrBadProposal)) + return + } + + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // Now update the upload status to trigger the correct pipeline + n, err := tx.Exec(`UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) if err != nil { - log.Errorw("failed to check if deal upload has started", "deal", id, "error", err) + log.Errorw("failed to finalize deal upload", "deal", id, "error", err) http.Error(w, "", int(ErrServerInternalError)) return } - if !valid { - log.Errorw("deal upload finalize failed", "deal", id, "error", "deal raw size does not match the sum of chunks") - http.Error(w, "deal raw size does not match the sum of chunks", int(ErrBadProposal)) - return + + if n == 0 { + return false, xerrors.Errorf("expected to update %d rows, got 0", n) } - } - // Now update the upload status to trigger the correct pipeline - n, err := m.DB.Exec(ctx, `UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) + _, err = tx.Exec(`DELETE FROM market_mk20_upload_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("failed to delete upload waiting: %w", err) + } + + if dealUpdated { + // Save the updated deal to DB + err = newDeal.UpdateDeal(tx) + if err != nil { + return false, xerrors.Errorf("failed to update deal: %w", err) + } + } + return true, nil + }) + if err != nil { log.Errorw("failed to finalize deal upload", "deal", id, "error", err) http.Error(w, "", int(ErrServerInternalError)) return } - if n == 0 { - log.Errorw("failed to finalize deal upload", "deal", id, "error", err) + if !comm { + log.Errorw("failed to finalize deal upload", "deal", id, "error", "failed to commit transaction") http.Error(w, "", int(ErrServerInternalError)) return } - w.WriteHeader(http.StatusOK) + w.WriteHeader(int(Ok)) } func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []ProductName, error) { @@ -543,10 +591,360 @@ func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []P return code, nil, nil, err } - // Save the updated deal to DB - err = ndeal.UpdateDeal(ctx, m.DB) + return Ok, ndeal, np, nil +} + +func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseWriter) { + ctx := context.Background() + var exists bool + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_upload_waiting + WHERE id = $1 AND chunked IS NULL);`, id.String()).Scan(&exists) if err != nil { - return ErrServerInternalError, nil, nil, xerrors.Errorf("failed to update deal: %w", err) + log.Errorw("failed to check if upload is waiting for data", "deal", id, "error", err) + w.WriteHeader(int(UploadServerError)) + return } - return Ok, ndeal, np, nil + if !exists { + http.Error(w, "deal not found", int(UploadStartCodeDealNotFound)) + return + } + + reader := NewTimeoutLimitReader(body, time.Second*5) + m.maxParallelUploads.Add(1) + + // Generate unique tmp pieceCID and Size for parked_pieces tables + wr := new(commp.Calc) + trSize, err := wr.Write([]byte(fmt.Sprintf("%s, %s", id.String(), time.Now().String()))) + if err != nil { + log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + digest, tsize, err := wr.Digest() + if err != nil { + panic(err) + } + + tpcid := cid.NewCidV1(cid.FilCommitmentUnsealed, digest) + var pnum, refID int64 + + // Generate piece park details with tmp pieceCID and Size + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.QueryRow(`SELECT id FROM parked_pieces + WHERE piece_cid = $1 + AND piece_padded_size = $2 + AND piece_raw_size = $3`, tpcid.String(), tsize, trSize).Scan(&pnum) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + err = tx.QueryRow(` + INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) + VALUES ($1, $2, $3, FALSE, TRUE) + ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING + RETURNING id`, tpcid.String(), tsize, trSize).Scan(&pnum) + if err != nil { + return false, xerrors.Errorf("inserting new parked piece and getting id: %w", err) + } + } else { + return false, xerrors.Errorf("checking existing parked piece: %w", err) + } + } + + // Add parked_piece_ref + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, FALSE) RETURNING ref_id`, pnum, "/PUT").Scan(&refID) + if err != nil { + return false, xerrors.Errorf("inserting parked piece ref: %w", err) + } + + return true, nil + }) + + if err != nil { + log.Errorw("failed to update chunk", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + + if !comm { + log.Errorw("failed to update chunk", "deal", id, "error", "failed to commit transaction") + http.Error(w, "", int(UploadServerError)) + return + } + + log.Debugw("tmp piece details generated for the chunk", "deal", id) + + failed := true + defer func() { + if failed { + _, err = m.DB.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + log.Errorw("failed to delete parked piece ref", "deal", id, "error", err) + } + } + }() + + // Store the piece and generate PieceCID and Size + pi, rawSize, err := m.sc.WriteUploadPiece(ctx, storiface.PieceNumber(pnum), UploadSizeLimit, reader, storiface.PathSealing, false) + if err != nil { + log.Errorw("failed to write piece", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + + log.Debugw("piece stored", "deal", id) + + // Update piece park details with correct values + comm, err = m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var pid int64 + // Check if we already have the piece, if found then verify access and skip rest of the processing + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCID.String(), pi.Size).Scan(&pid) + if err == nil { + // If piece exists then check if we can access the data + pr, err := m.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if err != nil { + // We should fail here because any subsequent operation which requires access to data will also fail + // till this error is fixed + if !errors.Is(err, storiface.ErrSectorNotFound) { + return false, fmt.Errorf("failed to get piece reader: %w", err) + } + + // If piece does not exist then we update piece park table to work with new tmpID + // Update ref table's reference to tmp id + _, err = tx.Exec(`UPDATE parked_piece_refs SET piece_id = $1 WHERE piece_id = $2`, pid, pnum) + if err != nil { + return false, xerrors.Errorf("updating parked piece ref: %w", err) + } + + // Now delete the original piece which has 404 error + _, err = tx.Exec(`DELETE FROM parked_pieces WHERE id = $1`, pid) + if err != nil { + return false, xerrors.Errorf("deleting parked piece: %w", err) + } + + // Update the tmp entry with correct details + n, err := tx.Exec(`UPDATE parked_pieces SET + piece_cid = $1, + piece_padded_size = $2, + piece_raw_size = $3, + complete = true + WHERE id = $4`, + pi.PieceCID.String(), pi.Size, rawSize, pnum) + if err != nil { + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + } else { + defer pr.Close() + // Add parked_piece_ref if no errors + var newRefID int64 + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, FALSE) RETURNING ref_id`, pid, "/PUT").Scan(&newRefID) + if err != nil { + return false, xerrors.Errorf("inserting parked piece ref: %w", err) + } + + // Remove the tmp refs. This will also delete the parked_pieces entry + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + return false, xerrors.Errorf("deleting tmp parked piece ref: %w", err) + } + // Update refID to be used later + refID = newRefID + } + } else { + if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("failed to check if piece already exists: %w", err) + } + // If piece does not exist then let's update the tmp one + n, err := tx.Exec(`UPDATE parked_pieces SET + piece_cid = $1, + piece_padded_size = $2, + piece_raw_size = $3, + complete = true + WHERE id = $4`, + pi.PieceCID.String(), pi.Size, rawSize, pnum) + if err != nil { + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + } + + n, err := tx.Exec(`UPDATE market_mk20_upload_waiting SET chunked = FALSE, ref_id = $2 WHERE id = $1`, id.String(), refID) + if err != nil { + return false, xerrors.Errorf("updating upload waiting: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating upload waiting: expected 1 row updated, got %d", n) + } + return true, nil + }) + + if err != nil { + log.Errorw("failed to update chunk", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + + if !comm { + log.Errorw("failed to update chunk", "deal", id, "error", "failed to commit transaction") + http.Error(w, "", int(UploadServerError)) + return + } + + log.Debugw("chunk upload finished", "deal", id) + + failed = false + w.WriteHeader(int(UploadOk)) +} + +func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { + ctx := context.Background() + var exists bool + err := m.DB.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 + FROM market_mk20_upload_waiting + WHERE id = $1 AND chunked = FALSE AND ref_id IS NOT NULL);`, id.String()).Scan(&exists) + if err != nil { + log.Errorw("failed to check if upload is waiting for data", "deal", id, "error", err) + w.WriteHeader(int(ErrServerInternalError)) + return + } + + if !exists { + http.Error(w, "deal not found", int(ErrDealNotFound)) + return + } + + ddeal, err := DealFromDB(ctx, m.DB, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + + if ddeal.Data == nil && deal == nil { + log.Errorw("cannot finalize deal with missing data source", "deal", id) + http.Error(w, "cannot finalize deal with missing data source", int(ErrBadProposal)) + return + } + + var pcidStr string + var rawSize, refID, pieceSize int64 + + err = m.DB.QueryRow(ctx, `SELECT r.ref_id, p.piece_cid, p.piece_padded_size, p.piece_raw_size + FROM market_mk20_upload_waiting u + JOIN parked_piece_refs r ON u.ref_id = r.ref_id + JOIN parked_pieces p ON r.piece_id = p.id + WHERE u.id = $1 + AND p.complete = TRUE + AND p.long_term = TRUE;`, id.String()).Scan(&refID, &pcidStr, &pieceSize, &rawSize) + if err != nil { + log.Errorw("failed to get piece details", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + pcid, err := cid.Parse(pcidStr) + if err != nil { + log.Errorw("failed to parse piece cid", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + } + + var uDeal *Deal + var dealUpdated bool + + if deal != nil { + // This is a deal where DataSource was not set - we should update the deal + code, ndeal, _, err := m.updateDealDetails(id, deal) + if err != nil { + log.Errorw("failed to update deal details", "deal", id, "error", err) + if code == ErrServerInternalError { + http.Error(w, "", int(ErrServerInternalError)) + } else { + http.Error(w, err.Error(), int(code)) + } + return + } + uDeal = ndeal + dealUpdated = true + } else { + uDeal = ddeal + } + + pi, err := uDeal.PieceInfo() + if err != nil { + log.Errorw("failed to get piece info", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + + if !pi.PieceCIDV1.Equals(pcid) { + log.Errorw("piece cid mismatch", "deal", id, "expected", pcid, "actual", pi.PieceCIDV1) + http.Error(w, "piece cid mismatch", int(ErrBadProposal)) + return + } + + if pi.Size != abi.PaddedPieceSize(pieceSize) { + log.Errorw("piece size mismatch", "deal", id, "expected", pi.Size, "actual", pieceSize) + http.Error(w, "piece size mismatch", int(ErrBadProposal)) + return + } + + if pi.RawSize != uint64(rawSize) { + log.Errorw("piece raw size mismatch", "deal", id, "expected", pi.RawSize, "actual", rawSize) + http.Error(w, "piece raw size mismatch", int(ErrBadProposal)) + return + } + + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + _, err = tx.Exec(`DELETE FROM market_mk20_upload_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("failed to delete upload waiting: %w", err) + } + + if dealUpdated { + // Save the updated deal to DB + err = uDeal.UpdateDeal(tx) + if err != nil { + return false, xerrors.Errorf("failed to update deal: %w", err) + } + } + + pdp := uDeal.Products.PDPV1 + + // Insert the PDP pipeline + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, + pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + } + return true, nil + }) + + if err != nil { + log.Errorw("failed to finalize deal upload", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + + if !comm { + log.Errorw("failed to finalize deal upload", "deal", id, "error", "failed to commit transaction") + http.Error(w, "", int(ErrServerInternalError)) + return + } + + w.WriteHeader(int(Ok)) } diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index f6108dfeb..c20c60f2c 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -171,23 +171,32 @@ func (m *MK20) Supported(ctx context.Context) (map[string]bool, map[string]bool, return productsMap, sourcesMap, nil } -type TimeoutReader struct { - r io.Reader - timeout time.Duration +type TimeoutLimitReader struct { + r io.Reader + timeout time.Duration + totalBytes int64 } -func NewTimeoutReader(r io.Reader, timeout time.Duration) *TimeoutReader { - return &TimeoutReader{ - r: r, - timeout: timeout, +func NewTimeoutLimitReader(r io.Reader, timeout time.Duration) *TimeoutLimitReader { + return &TimeoutLimitReader{ + r: r, + timeout: timeout, + totalBytes: 0, } } -func (t *TimeoutReader) Read(p []byte) (int, error) { +const UploadSizeLimit = int64(1 * 1024 * 1024) + +func (t *TimeoutLimitReader) Read(p []byte) (int, error) { deadline := time.Now().Add(t.timeout) for { // Attempt to read n, err := t.r.Read(p) + if t.totalBytes+int64(n) > UploadSizeLimit { + return 0, fmt.Errorf("upload size limit exceeded: %d bytes", UploadSizeLimit) + } else { + t.totalBytes += int64(n) + } if err != nil { return n, err diff --git a/market/mk20/utils.go b/market/mk20/utils.go index c1e4690e7..1f11645f5 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -271,6 +271,9 @@ type PieceInfo struct { } func (d *Deal) RawSize() (uint64, error) { + if d.Data == nil { + return 0, xerrors.Errorf("no data") + } commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) if err != nil { return 0, xerrors.Errorf("invalid piece cid: %w", err) @@ -279,6 +282,9 @@ func (d *Deal) RawSize() (uint64, error) { } func (d *Deal) Size() (abi.PaddedPieceSize, error) { + if d.Data == nil { + return 0, xerrors.Errorf("no data") + } commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) if err != nil { return 0, xerrors.Errorf("invalid piece cid: %w", err) @@ -493,13 +499,13 @@ func (d *Deal) UpdateDealWithTx(tx *harmonydb.Tx) error { return nil } -func (d *Deal) UpdateDeal(ctx context.Context, db *harmonydb.DB) error { +func (d *Deal) UpdateDeal(tx *harmonydb.Tx) error { dbDeal, err := d.ToDBDeal() if err != nil { return xerrors.Errorf("to db deal: %w", err) } - n, err := db.Exec(ctx, `UPDATE market_mk20_deal SET + n, err := tx.Exec(`UPDATE market_mk20_deal SET piece_cid_v2 = $1, piece_cid = $2, piece_size = $3, @@ -658,6 +664,9 @@ const ( // DealStateAccepted represents the state where a deal has been accepted and is pending further processing in the system. DealStateAccepted DealState = "accepted" + // DealStateAwaitingUpload represents the state where a deal is awaiting file upload to proceed further in the process. + DealStateAwaitingUpload DealState = "uploading" + // DealStateProcessing represents the state of a deal currently being processed in the pipeline. DealStateProcessing DealState = "processing" diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index 6c5012c89..414ad6d95 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -2,6 +2,7 @@ package pdp import ( "context" + "errors" "fmt" "io" "math/bits" @@ -9,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-data-segment/datasegment" @@ -183,12 +185,35 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b var pieceParked bool comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - err = tx.QueryRow(` + // Check if we already have the piece, if found then verify access and skip rest of the processing + var pid int64 + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCIDV1.String(), pi.Size).Scan(&pid) + if err == nil { + // If piece exists then check if we can access the data + pr, err := a.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if err != nil { + // If piece does not exist then we will park it otherwise fail here + if !errors.Is(err, storiface.ErrSectorNotFound) { + // We should fail here because any subsequent operation which requires access to data will also fail + // till this error is fixed + return false, fmt.Errorf("failed to get piece reader: %w", err) + } + } + defer pr.Close() + pieceParked = true + parkedPieceID = pid + } else { + if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("failed to check if piece already exists: %w", err) + } + // If piece does not exist then let's create one + err = tx.QueryRow(` INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) - VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, - pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID, &pieceParked) - if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id`, + pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } } err = tx.QueryRow(` @@ -224,7 +249,7 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b // Write piece if not already complete if !pieceParked { - upi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage) + upi, _, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage, true) if err != nil { return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) } diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index fa5689fb6..677ff3719 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -129,12 +129,35 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo var pieceParked bool comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - err = tx.QueryRow(` + // Check if we already have the piece, if found then verify access and skip rest of the processing + var pid int64 + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pcid.String(), psize).Scan(&pid) + if err == nil { + // If piece exists then check if we can access the data + pr, err := a.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if err != nil { + // If piece does not exist then we will park it otherwise fail here + if !errors.Is(err, storiface.ErrSectorNotFound) { + // We should fail here because any subsequent operation which requires access to data will also fail + // till this error is fixed + return false, fmt.Errorf("failed to get piece reader: %w", err) + } + } + defer pr.Close() + pieceParked = true + parkedPieceID = pid + } else { + if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("failed to check if piece already exists: %w", err) + } + // If piece does not exist then let's create one + err = tx.QueryRow(` INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) - VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, - pcid.String(), psize, rawSize).Scan(&parkedPieceID, &pieceParked) - if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id`, + pcid.String(), psize, rawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } } err = tx.QueryRow(` @@ -181,19 +204,19 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo // Write piece if not already complete if !pieceParked { - pi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), rawSize, rd, storiface.PathStorage) + cpi, _, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), rawSize, rd, storiface.PathStorage, true) if err != nil { return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) } - if !pi.PieceCID.Equals(pcid) { + if !cpi.PieceCID.Equals(pcid) { cleanupChunks = true - return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", pi.PieceCID.String(), pcid.String()) + return false, xerrors.Errorf("commP mismatch calculated %s and supplied %s", cpi.PieceCID.String(), pcid.String()) } - if pi.Size != psize { + if cpi.Size != psize { cleanupChunks = true - return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", pi.Size, psize) + return false, xerrors.Errorf("commP size mismatch calculated %d and supplied %d", cpi.Size, psize) } } diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index df98e2501..cb4118ef2 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -72,7 +72,6 @@ type MK20PipelinePiece struct { } func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { - go d.pipelineInsertLoop(ctx) // Catch any panics if encountered as we are working with user provided data defer func() { if r := recover(); r != nil { diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index de0aeb8b8..5c9089544 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -2,6 +2,7 @@ package storage_market import ( "context" + "errors" "fmt" "io" "math/bits" @@ -10,6 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-data-segment/datasegment" @@ -205,12 +207,35 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool var pieceParked bool comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - err = tx.QueryRow(` + // Check if we already have the piece, if found then verify access and skip rest of the processing + var pid int64 + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCIDV1.String(), pi.Size).Scan(&pid) + if err == nil { + // If piece exists then check if we can access the data + pr, err := a.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if err != nil { + // If piece does not exist then we will park it otherwise fail here + if !errors.Is(err, storiface.ErrSectorNotFound) { + // We should fail here because any subsequent operation which requires access to data will also fail + // till this error is fixed + return false, fmt.Errorf("failed to get piece reader: %w", err) + } + } + defer pr.Close() + pieceParked = true + parkedPieceID = pid + } else { + if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("failed to check if piece already exists: %w", err) + } + // If piece does not exist then let's create one + err = tx.QueryRow(` INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) - VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id, complete`, - pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID, &pieceParked) - if err != nil { - return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + VALUES ($1, $2, $3, TRUE, TRUE) RETURNING id`, + pi.PieceCIDV1.String(), pi.Size, pi.RawSize).Scan(&parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to create parked_pieces entry: %w", err) + } } err = tx.QueryRow(` @@ -246,7 +271,7 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool // Write piece if not already complete if !pieceParked { - upi, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage) + upi, _, err := a.sc.WriteUploadPiece(ctx, storiface.PieceNumber(parkedPieceID), int64(pi.RawSize), outR, storiface.PathStorage, true) if err != nil { return false, xerrors.Errorf("writing aggregated piece data to storage: %w", err) } From 1587e740407b8cb994baca90f3441c9777fd9954 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 8 Aug 2025 16:40:02 +0400 Subject: [PATCH 23/55] working PDP pipeline, indexing, ipni --- Dockerfile | 16 +- cmd/curio/tasks/tasks.go | 27 +- cmd/sptool/toolbox_deal_client.go | 546 +++++++++++++++++- docker/piece-server/sample/pdp.sh | 54 ++ documentation/en/curio-cli/sptool.md | 54 +- .../sql/20240731-market-migration.sql | 12 +- harmony/harmonydb/sql/20240823-ipni.sql | 2 + .../harmonydb/sql/20250505-market_mk20.sql | 92 ++- market/ipni/ipni-provider/ipni-provider.go | 7 +- market/ipni/ipni-provider/spark.go | 3 + market/mk20/client/auth.go | 42 ++ market/mk20/client/client.go | 404 +++++++++++++ market/mk20/client/http_client.go | 235 ++++++++ market/mk20/ddo_v1.go | 34 +- market/mk20/http/docs.go | 175 +++++- market/mk20/http/http.go | 44 +- market/mk20/http/swagger.json | 175 +++++- market/mk20/http/swagger.yaml | 143 ++++- market/mk20/mk20.go | 125 +++- market/mk20/mk20_upload.go | 7 +- market/mk20/pdp_v1.go | 73 +-- market/mk20/retrieval_v1.go | 4 + market/mk20/types.go | 22 +- market/mk20/utils.go | 57 +- pdp/contract/addresses.go | 21 +- tasks/gc/pipeline_meta_gc.go | 12 + tasks/gc/storage_gc_mark.go | 3 + tasks/indexing/task_check_indexes.go | 4 +- tasks/indexing/task_indexing.go | 6 - tasks/indexing/task_ipni.go | 4 +- tasks/indexing/task_pdp_indexing.go | 361 ++++++++++++ tasks/indexing/task_pdp_ipni.go | 508 ++++++++++++++++ tasks/pdp/proofset_addroot_watch.go | 74 +-- tasks/pdp/proofset_create_watch.go | 20 +- tasks/pdp/proofset_delete_root_watch.go | 52 +- tasks/pdp/proofset_delete_watch.go | 52 +- tasks/pdp/task_add_proofset.go | 36 +- tasks/pdp/task_addroot.go | 78 +-- tasks/pdp/task_aggregation.go | 11 +- tasks/pdp/task_delete_root.go | 52 +- tasks/pdp/task_delete_rootset.go | 42 +- tasks/pdp/task_prove.go | 2 +- tasks/pdp/task_save_cache.go | 18 +- tasks/piece/task_aggregate_chunks.go | 13 +- web/api/webrpc/market.go | 5 +- web/api/webrpc/market_20.go | 4 +- 46 files changed, 3305 insertions(+), 426 deletions(-) create mode 100755 docker/piece-server/sample/pdp.sh create mode 100644 market/mk20/client/auth.go create mode 100644 market/mk20/client/client.go create mode 100644 market/mk20/client/http_client.go create mode 100644 tasks/indexing/task_pdp_indexing.go create mode 100644 tasks/indexing/task_pdp_ipni.go diff --git a/Dockerfile b/Dockerfile index 2e9f131a9..66216d680 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,10 +61,23 @@ RUN go install github.com/LexLuthr/piece-server@latest \ RUN go install github.com/ipni/storetheindex@v0.8.38 \ && cp $GOPATH/bin/storetheindex /usr/local/bin/ +RUN go install github.com/ethereum/go-ethereum/cmd/geth@latest \ + && cp $GOPATH/bin/geth /usr/local/bin/ + ##################################### FROM ubuntu:22.04 AS curio-all-in-one -RUN apt-get update && apt-get install -y dnsutils vim curl aria2 jq +RUN apt-get update && apt-get install -y dnsutils vim curl aria2 jq git wget nodejs npm + +# Install Foundry +RUN curl -L https://foundry.paradigm.xyz | bash \ + && bash -c ". ~/.foundry/bin/foundryup" + +# Make sure foundry binaries are available in PATH +ENV PATH="/root/.foundry/bin:${PATH}" + +# Verify installation +RUN forge --version && cast --version && anvil --version # Copy libraries and binaries from curio-builder COPY --from=curio-builder /etc/ssl/certs /etc/ssl/certs @@ -100,6 +113,7 @@ COPY --from=curio-builder /opt/curio/sptool /usr/local/bin/ COPY --from=piece-server-builder /usr/local/bin/piece-server /usr/local/bin/ COPY --from=piece-server-builder /usr/local/bin/car /usr/local/bin/ COPY --from=piece-server-builder /usr/local/bin/storetheindex /usr/local/bin/ +COPY --from=piece-server-builder /usr/local/bin/geth /usr/local/bin/ # Set up directories and permissions RUN mkdir /var/tmp/filecoin-proof-parameters \ diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index b744f6432..da4e2e9c8 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -287,26 +287,33 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan sdeps.EthSender = es pdp.NewWatcherCreate(db, must.One(dependencies.EthClient.Val()), chainSched) - pdp.NewWatcherRootAdd(db, must.One(dependencies.EthClient.Val()), chainSched) - pdp.NewWatcherDelete(db, must.One(dependencies.EthClient.Val()), chainSched) - pdp.NewPDPTaskDeleteRoot(db, es, must.One(dependencies.EthClient.Val())) - pdp.NewWatcherRootDelete(db, must.One(dependencies.EthClient.Val()), chainSched) + pdp.NewWatcherRootAdd(db, chainSched) + pdp.NewWatcherDelete(db, chainSched) + pdp.NewWatcherRootDelete(db, chainSched) - pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) - pdpCache := pdp.NewTaskSavePDPCache(db, dependencies.CachedPieceReader, iStore) - pdpAddRoot := pdp.NewPDPTaskAddRoot(db, es, must.One(dependencies.EthClient.Val())) pdpProveTask := pdp.NewProveTask(chainSched, db, must.One(dependencies.EthClient.Val()), dependencies.Chain, es, dependencies.CachedPieceReader, iStore) pdpNextProvingPeriodTask := pdp.NewNextProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpInitProvingPeriodTask := pdp.NewInitProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpNotifTask := pdp.NewPDPNotifyTask(db) - activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, pdpAddRoot, pdpAggregateTask, pdpCache) + + addProofSetTask := pdp.NewPDPTaskAddProofSet(db, es, must.One(dependencies.EthClient.Val()), full) + pdpAddRoot := pdp.NewPDPTaskAddRoot(db, es, must.One(dependencies.EthClient.Val())) + pdpDelRoot := pdp.NewPDPTaskDeleteRoot(db, es, must.One(dependencies.EthClient.Val())) + pdpDelProofSetTask := pdp.NewPDPTaskDeleteProofSet(db, es, must.One(dependencies.EthClient.Val()), full) + + pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) + pdpCache := pdp.NewTaskSavePDPCache(db, dependencies.CachedPieceReader, iStore) + + activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, pdpAddRoot, addProofSetTask, pdpAggregateTask, pdpCache, pdpDelRoot, pdpDelProofSetTask) } idxMax := taskhelp.Max(cfg.Subsystems.IndexingMaxTasks) indexingTask := indexing.NewIndexingTask(db, sc, iStore, dependencies.SectorReader, dependencies.CachedPieceReader, cfg, idxMax) - ipniTask := indexing.NewIPNITask(db, sc, iStore, dependencies.SectorReader, dependencies.CachedPieceReader, cfg, idxMax) - activeTasks = append(activeTasks, ipniTask, indexingTask) + ipniTask := indexing.NewIPNITask(db, sc, dependencies.SectorReader, dependencies.CachedPieceReader, cfg, idxMax) + pdpIdxTask := indexing.NewPDPIndexingTask(db, sc, iStore, dependencies.CachedPieceReader, cfg, idxMax) + pdpIPNITask := indexing.NewPDPIPNITask(db, sc, dependencies.CachedPieceReader, cfg, idxMax) + activeTasks = append(activeTasks, ipniTask, indexingTask, pdpIdxTask, pdpIPNITask) if cfg.HTTP.Enable { err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps) diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 385e6b60e..04347731d 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -1580,8 +1580,10 @@ var mk20Clientcmd = &cli.Command{ initCmd, comm2Cmd, mk20DealCmd, + mk20PDPDealCmd, mk20ClientMakeAggregateCmd, mk20ClientUploadCmd, + mk20ClientChunkUploadCmd, }, } @@ -2008,9 +2010,9 @@ var mk20ClientMakeAggregateCmd = &cli.Command{ }, } -var mk20ClientUploadCmd = &cli.Command{ - Name: "upload", - Usage: "Upload a file to the storage provider", +var mk20ClientChunkUploadCmd = &cli.Command{ + Name: "chunk-upload", + Usage: "Upload a file in chunks to the storage provider", Flags: []cli.Flag{ &cli.StringFlag{ Name: "provider", @@ -2147,7 +2149,7 @@ var mk20ClientUploadCmd = &cli.Command{ return err } log.Debugw("request body", "body", string(b)) - client, err := http.NewRequest("POST", purl.String()+"/market/mk20/upload/"+dealid.String(), bytes.NewBuffer(b)) + client, err := http.NewRequest("POST", purl.String()+"/market/mk20/uploads/"+dealid.String(), bytes.NewBuffer(b)) if err != nil { return xerrors.Errorf("failed to upload start create request: %w", err) } @@ -2172,7 +2174,7 @@ var mk20ClientUploadCmd = &cli.Command{ defer x.Close() for { - resp, err = http.Get(purl.String() + "/market/mk20/upload/" + dealid.String()) + resp, err = http.Get(purl.String() + "/market/mk20/uploads/" + dealid.String()) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } @@ -2216,7 +2218,7 @@ var mk20ClientUploadCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to read chunk: %w", err) } - req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/upload/"+dealid.String()+"/"+fmt.Sprintf("%d", c), bytes.NewBuffer(buf)) + req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/uploads/"+dealid.String()+"/"+fmt.Sprintf("%d", c), bytes.NewBuffer(buf)) if err != nil { return xerrors.Errorf("failed to create put request: %w", err) } @@ -2239,7 +2241,537 @@ var mk20ClientUploadCmd = &cli.Command{ log.Infow("upload complete") //Finalize the upload - resp, err = http.Post(purl.String()+"/market/mk20/upload/finalize/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) + resp, err = http.Post(purl.String()+"/market/mk20/uploads/finalize/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) + if err != nil { + return xerrors.Errorf("failed to send request: %w", err) + } + if resp.StatusCode != http.StatusOK { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + + return nil + }, +} + +var mk20PDPDealCmd = &cli.Command{ + Name: "pdp-deal", + Usage: "Make a mk20 PDP deal with Curio", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "http-url", + Usage: "http url to CAR file", + }, + &cli.StringSliceFlag{ + Name: "http-headers", + Usage: "http headers to be passed with the request (e.g key=value)", + }, + &cli.StringFlag{ + Name: "provider", + Usage: "storage provider on-chain address", + Required: true, + }, + &cli.StringFlag{ + Name: "pcidv2", + Usage: "pcidv2 of the CAR file", + }, + &cli.StringFlag{ + Name: "wallet", + Usage: "wallet address to be used to initiate the deal", + }, + &cli.StringFlag{ + Name: "aggregate", + Usage: "aggregate file path for the deal", + }, + &cli.BoolFlag{ + Name: "put", + Usage: "used HTTP put as data source", + }, + &cli.BoolFlag{ + Name: "add-root", + Usage: "add root", + }, + &cli.BoolFlag{ + Name: "add-proofset", + Usage: "add proofset", + }, + &cli.BoolFlag{ + Name: "remove-root", + Usage: "remove root", + }, + &cli.BoolFlag{ + Name: "remove-proofset", + Usage: "remove proofset", + }, + &cli.StringFlag{ + Name: "record-keeper", + Usage: "record keeper address", + }, + &cli.Uint64SliceFlag{ + Name: "root-id", + Usage: "root IDs", + }, + &cli.Uint64Flag{ + Name: "proofset-id", + Usage: "proofset IDs", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + n, err := Setup(cctx.String(mk12_client_repo.Name)) + if err != nil { + return err + } + + api, closer, err := lcli.GetGatewayAPIV1(cctx) + if err != nil { + return fmt.Errorf("cant setup gateway connection: %w", err) + } + defer closer() + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return err + } + + log.Debugw("selected wallet", "wallet", walletAddr) + + maddr, err := address.NewFromString(cctx.String("provider")) + if err != nil { + return err + } + + minfo, err := api.StateMinerInfo(ctx, maddr, chain_types.EmptyTSK) + if err != nil { + return err + } + if minfo.PeerId == nil { + return xerrors.Errorf("storage provider %s has no peer ID set on-chain", maddr) + } + + var maddrs []multiaddr.Multiaddr + for _, mma := range minfo.Multiaddrs { + ma, err := multiaddr.NewMultiaddrBytes(mma) + if err != nil { + return xerrors.Errorf("storage provider %s had invalid multiaddrs in their info: %w", maddr, err) + } + maddrs = append(maddrs, ma) + } + if len(maddrs) == 0 { + return xerrors.Errorf("storage provider %s has no multiaddrs set on-chain", maddr) + } + + addrInfo := &peer.AddrInfo{ + ID: *minfo.PeerId, + Addrs: maddrs, + } + + log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr) + + var hurls []*url.URL + + for _, ma := range addrInfo.Addrs { + hurl, err := maurl.ToURL(ma) + if err != nil { + return xerrors.Errorf("failed to convert multiaddr %s to URL: %w", ma, err) + } + if hurl.Scheme == "ws" { + hurl.Scheme = "http" + } + if hurl.Scheme == "wss" { + hurl.Scheme = "https" + } + log.Debugw("converted multiaddr to URL", "url", hurl, "multiaddr", ma.String()) + hurls = append(hurls, hurl) + } + + addRoot := cctx.Bool("add-root") + addProofset := cctx.Bool("add-proofset") + removeRoot := cctx.Bool("remove-root") + removeProofset := cctx.Bool("remove-proofset") + recordKeeper := cctx.String("record-keeper") + rootIDs := cctx.Uint64Slice("root-id") + proofSetSet := cctx.IsSet("proofset-id") + proofsetID := cctx.Uint64("proofset-id") + if !(addRoot || removeRoot || addProofset || removeProofset) { + return xerrors.Errorf("at least one of --add-root, --remove-root, --add-proofset, --remove-proofset must be set") + } + + if btoi(addRoot)+btoi(addProofset)+btoi(removeRoot)+btoi(removeProofset) > 1 { + return xerrors.Errorf("only one of --add-root, --remove-root, --add-proofset, --remove-proofset can be set") + } + + extraBytes := []byte{} + + pdp := &mk20.PDPV1{} + var d *mk20.DataSource + var ret *mk20.RetrievalV1 + + if addRoot { + commp := cctx.String("pcidv2") + pieceCid, err := cid.Parse(commp) + if err != nil { + return xerrors.Errorf("parsing pcidv2 '%s': %w", commp, err) + } + + var headers http.Header + + for _, header := range cctx.StringSlice("http-headers") { + sp := strings.Split(header, "=") + if len(sp) != 2 { + return xerrors.Errorf("malformed http header: %s", header) + } + headers.Add(sp[0], sp[1]) + } + + if cctx.IsSet("aggregate") { + d = &mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Aggregate: &mk20.FormatAggregate{ + Type: mk20.AggregateTypeV1, + }, + }, + } + + var pieces []mk20.DataSource + + log.Debugw("using aggregate data source", "aggregate", cctx.String("aggregate")) + // Read file line by line + loc, err := homedir.Expand(cctx.String("aggregate")) + if err != nil { + return err + } + file, err := os.Open(loc) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Split(line, "\t") + if len(parts) != 2 { + return fmt.Errorf("invalid line format. Expected pieceCidV2, url at %s", line) + } + if parts[0] == "" || parts[1] == "" { + return fmt.Errorf("empty column value in the input file at %s", line) + } + + pieceCid, err := cid.Parse(parts[0]) + if err != nil { + return fmt.Errorf("failed to parse CID: %w", err) + } + + url, err := url.Parse(parts[1]) + if err != nil { + return fmt.Errorf("failed to parse url: %w", err) + } + + pieces = append(pieces, mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Priority: 0, + Fallback: true, + }, + }, + }, + }) + + if err := scanner.Err(); err != nil { + return err + } + } + d.SourceAggregate = &mk20.DataSourceAggregate{ + Pieces: pieces, + } + } else { + if !cctx.IsSet("http-url") { + if cctx.Bool("put") { + d = &mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHttpPut: &mk20.DataSourceHttpPut{}, + } + } else { + d = &mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceOffline: &mk20.DataSourceOffline{}, + } + } + } else { + url, err := url.Parse(cctx.String("http-url")) + if err != nil { + return xerrors.Errorf("parsing http url: %w", err) + } + d = &mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Headers: headers, + Priority: 0, + Fallback: true, + }, + }, + }, + } + } + } + + if !proofSetSet { + return xerrors.Errorf("proofset-id must be set when adding a root") + } + pdp.AddRoot = true + pdp.RecordKeeper = recordKeeper + pdp.ProofSetID = &proofsetID + pdp.ExtraData = extraBytes + ret = &mk20.RetrievalV1{ + Indexing: true, + AnnouncePayload: true, + } + } + + if removeRoot { + if !proofSetSet { + return xerrors.Errorf("proofset-id must be set when removing a root") + } + pdp.DeleteRoot = true + pdp.RecordKeeper = recordKeeper + pdp.ProofSetID = &proofsetID + pdp.RootIDs = rootIDs + pdp.ExtraData = extraBytes + d = nil + } + + if addProofset { + pdp.RecordKeeper = recordKeeper + pdp.CreateProofSet = true + pdp.ExtraData = extraBytes + d = nil + } + + if removeProofset { + if !proofSetSet { + return xerrors.Errorf("proofset-id must be set when deleting proof-set") + } + pdp.RecordKeeper = recordKeeper + pdp.DeleteProofSet = true + pdp.ProofSetID = &proofsetID + pdp.ExtraData = extraBytes + d = nil + } + + p := mk20.Products{ + PDPV1: pdp, + } + + if ret != nil { + p.RetrievalV1 = ret + } + + id, err := mk20.NewULID() + if err != nil { + return err + } + log.Debugw("generated deal id", "id", id) + + deal := mk20.Deal{ + Identifier: id, + Client: walletAddr, + Products: p, + } + + if d != nil { + deal.Data = d + } + + log.Debugw("deal", "deal", deal) + + body, err := json.Marshal(deal) + if err != nil { + return err + } + + // Try to request all URLs one by one and exit after first success + for _, u := range hurls { + s := u.String() + "/market/mk20/store" + log.Debugw("trying to send request to", "url", u.String()) + req, err := http.NewRequest("POST", s, bytes.NewReader(body)) + if err != nil { + return xerrors.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + log.Debugw("Headers", "headers", req.Header) + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Warnw("failed to send request", "url", s, "error", err) + continue + } + if resp.StatusCode != http.StatusOK { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + log.Warnw("failed to send request", "url", s, "status", resp.StatusCode, "body", string(respBody)) + continue + } + return nil + } + return xerrors.Errorf("failed to send request to any of the URLs") + }, +} + +func btoi(b bool) int { + if b { + return 1 + } + return 0 +} + +var mk20ClientUploadCmd = &cli.Command{ + Name: "upload", + Usage: "Upload a file to the storage provider", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "provider", + Usage: "storage provider on-chain address", + Required: true, + }, + &cli.StringFlag{ + Name: "deal", + Usage: "deal id to upload to", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 1 { + return xerrors.Errorf("must provide a single file to upload") + } + file := cctx.Args().First() + log.Debugw("uploading file", "file", file) + ctx := cctx.Context + + dealid, err := ulid.Parse(cctx.String("deal")) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + + maddr, err := address.NewFromString(cctx.String("provider")) + if err != nil { + return err + } + + f, err := os.OpenFile(file, os.O_RDONLY, 0644) + if err != nil { + return xerrors.Errorf("opening file: %w", err) + } + + stat, err := f.Stat() + if err != nil { + return xerrors.Errorf("stat file: %w", err) + } + + size := stat.Size() + if size == 0 { + return xerrors.Errorf("file size is 0") + } + + f.Close() + + api, closer, err := lcli.GetGatewayAPIV1(cctx) + if err != nil { + return fmt.Errorf("cant setup gateway connection: %w", err) + } + defer closer() + + minfo, err := api.StateMinerInfo(ctx, maddr, chain_types.EmptyTSK) + if err != nil { + return err + } + if minfo.PeerId == nil { + return xerrors.Errorf("storage provider %s has no peer ID set on-chain", maddr) + } + + var maddrs []multiaddr.Multiaddr + for _, mma := range minfo.Multiaddrs { + ma, err := multiaddr.NewMultiaddrBytes(mma) + if err != nil { + return xerrors.Errorf("storage provider %s had invalid multiaddrs in their info: %w", maddr, err) + } + maddrs = append(maddrs, ma) + } + if len(maddrs) == 0 { + return xerrors.Errorf("storage provider %s has no multiaddrs set on-chain", maddr) + } + + addrInfo := &peer.AddrInfo{ + ID: *minfo.PeerId, + Addrs: maddrs, + } + + log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr) + + var hurls []*url.URL + + for _, ma := range addrInfo.Addrs { + hurl, err := maurl.ToURL(ma) + if err != nil { + return xerrors.Errorf("failed to convert multiaddr %s to URL: %w", ma, err) + } + if hurl.Scheme == "ws" { + hurl.Scheme = "http" + } + if hurl.Scheme == "wss" { + hurl.Scheme = "https" + } + log.Debugw("converted multiaddr to URL", "url", hurl, "multiaddr", ma.String()) + hurls = append(hurls, hurl) + } + + purl := hurls[0] + log.Debugw("using first URL", "url", purl) + + req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/upload/"+dealid.String(), f) + if err != nil { + return xerrors.Errorf("failed to create put request: %w", err) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("failed to send put request: %w", err) + } + if resp.StatusCode != http.StatusOK { + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return xerrors.Errorf("failed to read response body: %w", err) + } + return xerrors.Errorf("failed to send request: %d, %s", resp.StatusCode, string(respBody)) + } + + log.Infow("upload complete") + + //Finalize the upload + resp, err = http.Post(purl.String()+"/market/mk20/upload/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } diff --git a/docker/piece-server/sample/pdp.sh b/docker/piece-server/sample/pdp.sh new file mode 100755 index 000000000..aee058635 --- /dev/null +++ b/docker/piece-server/sample/pdp.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -e + +ci="\e[3m" +cn="\e[0m" + + +put="${1:-false}" +offline="${2:-false}" +chunks="${3:-51200}" +links="${4:-100}" + +printf "${ci}sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'\n\n${cn}" +FILE=`sptool --actor t01000 toolbox mk12-client generate-rand-car -c=$chunks -l=$links -s=5120000 /var/lib/curio-client/data/ | awk '{print $NF}'` +read COMMP_CID PIECE CAR < <(sptool --actor t01000 toolbox mk20-client commp $FILE 2>/dev/null | awk -F': ' '/CID/ {cid=$2} /Piece/ {piece=$2} /Car/ {car=$2} END {print cid, piece, car}') + +mv $FILE /var/lib/curio-client/data/$COMMP_CID + +miner_actor=$(lotus state list-miners | grep -v t01000) +printf "$COMMP_CID\n" +printf "$PIECE\n" +printf "$CAR\n" + +#if [ "$put" == "true" ]; then +# ################################################################################### +# printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +# --pcidv2=$COMMP_CID \ +# --contract-address 0xtest --contract-verify-method test --put\n\n${cn}" +# +# sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test --put +# +#else +# +# if [ "$offline" == "true" ]; then +# +# ################################################################################### +# printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +# --pcidv2=$COMMP_CID \ +# --contract-address 0xtest --contract-verify-method test\n\n${cn}" +# +# sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test +# +# else +# ################################################################################### +# printf "${ci}sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor \ +# --http-url=http://piece-server:12320/pieces?id=$COMMP_CID \ +# --pcidv2=$COMMP_CID \ +# --contract-address 0xtest --contract-verify-method test\n\n${cn}" +# +# sptool --actor t01000 toolbox mk20-client deal --provider=$miner_actor --http-url=http://piece-server:12320/pieces?id=$COMMP_CID --pcidv2=$COMMP_CID --contract-address 0xtest --contract-verify-method test +# +# fi +# +#fi diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index a957db255..57b1f3395 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -899,12 +899,14 @@ USAGE: sptool toolbox mk20-client command [command options] COMMANDS: - init Initialise curio mk12 client repo - commp - deal Make a mk20 deal with Curio - aggregate Create a new aggregate from a list of CAR files - upload Upload a file to the storage provider - help, h Shows a list of commands or help for one command + init Initialise curio mk12 client repo + commp + deal Make a mk20 deal with Curio + pdp-deal Make a mk20 PDP deal with Curio + aggregate Create a new aggregate from a list of CAR files + upload Upload a file to the storage provider + chunk-upload Upload a file in chunks to the storage provider + help, h Shows a list of commands or help for one command OPTIONS: --mk12-client-repo value repo directory for mk12 client (default: "~/.curio-client") [$CURIO_MK12_CLIENT_REPO] @@ -960,6 +962,32 @@ OPTIONS: --help, -h show help ``` +#### sptool toolbox mk20-client pdp-deal +``` +NAME: + sptool toolbox mk20-client pdp-deal - Make a mk20 PDP deal with Curio + +USAGE: + sptool toolbox mk20-client pdp-deal [command options] + +OPTIONS: + --http-url value http url to CAR file + --http-headers value [ --http-headers value ] http headers to be passed with the request (e.g key=value) + --provider value storage provider on-chain address + --pcidv2 value pcidv2 of the CAR file + --wallet value wallet address to be used to initiate the deal + --aggregate value aggregate file path for the deal + --put used HTTP put as data source (default: false) + --add-root add root (default: false) + --add-proofset add proofset (default: false) + --remove-root remove root (default: false) + --remove-proofset remove proofset (default: false) + --record-keeper value record keeper address + --root-id value [ --root-id value ] root IDs + --proofset-id value proofset IDs (default: 0) + --help, -h show help +``` + #### sptool toolbox mk20-client aggregate ``` NAME: @@ -983,6 +1011,20 @@ NAME: USAGE: sptool toolbox mk20-client upload [command options] +OPTIONS: + --provider value storage provider on-chain address + --deal value deal id to upload to + --help, -h show help +``` + +#### sptool toolbox mk20-client chunk-upload +``` +NAME: + sptool toolbox mk20-client chunk-upload - Upload a file in chunks to the storage provider + +USAGE: + sptool toolbox mk20-client chunk-upload [command options] + OPTIONS: --provider value storage provider on-chain address --deal value deal id to upload to diff --git a/harmony/harmonydb/sql/20240731-market-migration.sql b/harmony/harmonydb/sql/20240731-market-migration.sql index 9ee7a71ed..ae57c3dac 100644 --- a/harmony/harmonydb/sql/20240731-market-migration.sql +++ b/harmony/harmonydb/sql/20240731-market-migration.sql @@ -25,6 +25,7 @@ CREATE TABLE market_mk12_deals ( piece_cid TEXT NOT NULL, piece_size BIGINT NOT NULL, + -- raw_size BIGINT (Added in 20250505-market_mk20.sql) fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, @@ -54,6 +55,8 @@ CREATE TABLE market_piece_metadata ( indexed BOOLEAN NOT NULL DEFAULT FALSE, indexed_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + -- dropped in 20250505-market_mk20.sql + -- PRIMARY KEY (piece_cid, piece_size) (Added in 20250505-market_mk20.sql) constraint market_piece_meta_identity_key unique (piece_cid, piece_size) ); @@ -64,7 +67,6 @@ CREATE TABLE market_piece_metadata ( -- Cleanup for this table will be created in a later stage. CREATE TABLE market_piece_deal ( id TEXT NOT NULL, -- (UUID for new deals, PropCID for old) - piece_cid TEXT NOT NULL, boost_deal BOOLEAN NOT NULL, legacy_deal BOOLEAN NOT NULL DEFAULT FALSE, @@ -73,11 +75,16 @@ CREATE TABLE market_piece_deal ( sp_id BIGINT NOT NULL, sector_num BIGINT NOT NULL, + piece_offset BIGINT NOT NULL, -- NOT NULL dropped in 20250505-market_mk20.sql + + -- piece_ref BIGINT (Added in 20250505-market_mk20.sql) - piece_offset BIGINT NOT NULL, + piece_cid TEXT NOT NULL, piece_length BIGINT NOT NULL, raw_size BIGINT NOT NULL, + -- Dropped both constraint and primary key in 20250505-market_mk20.sql + -- ADD PRIMARY KEY (id, sp_id, piece_cid, piece_length) (Added in 20250505-market_mk20.sql) primary key (sp_id, piece_cid, id), constraint market_piece_deal_identity_key unique (sp_id, id) @@ -227,6 +234,7 @@ CREATE TABLE market_direct_deals ( piece_cid TEXT NOT NULL, piece_size BIGINT NOT NULL, + -- raw_size BIGINT (Added in 20250505-market_mk20.sql) fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index b9b9117c2..767caa80a 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -87,6 +87,8 @@ CREATE TABLE ipni_task ( task_id BIGINT DEFAULT NULL, complete BOOLEAN DEFAULT FALSE, + -- id TEXT (Added in 20250505-market_mk20.sql) + PRIMARY KEY (provider, context_id, is_rm) ); diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index fa8214ffc..e3a0a1292 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -28,12 +28,16 @@ DROP CONSTRAINT IF EXISTS market_piece_deal_identity_key; -- Add the new composite primary key for market_piece_deal ALTER TABLE market_piece_deal - ADD PRIMARY KEY (sp_id, id, piece_cid, piece_length); + ADD PRIMARY KEY (id, sp_id, piece_cid, piece_length); -- Add a column to relate a piece park piece to mk20 deal ALTER TABLE market_piece_deal ADD COLUMN piece_ref BIGINT; +-- Allow piece_offset to be null for PDP deals +ALTER TABLE market_piece_deal + ALTER COLUMN piece_offset DROP NOT NULL; + -- Add column to skip scheduling piece_park. Used for upload pieces ALTER TABLE parked_pieces ADD COLUMN skip BOOLEAN NOT NULL DEFAULT FALSE; @@ -72,7 +76,7 @@ BEGIN ) VALUES ( _id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id, _sp_id, _sector_num, _piece_offset, _piece_length, _raw_size, _piece_ref - ) ON CONFLICT (sp_id, id, piece_cid, piece_length) DO NOTHING; + ) ON CONFLICT (id, sp_id, piece_cid, piece_length) DO NOTHING; END; $$ LANGUAGE plpgsql; @@ -105,9 +109,9 @@ BEGIN -- If a different is_rm exists for the same context_id and provider, insert the new task IF FOUND THEN - INSERT INTO ipni_task (sp_id, id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) - VALUES (_sp_id, _id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); - RETURN; + INSERT INTO ipni_task (sp_id, id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); + RETURN; END IF; -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table @@ -436,7 +440,7 @@ CREATE TABLE pdp_proof_set_delete ( ); -- This table governs the delete root tasks -CREATE TABLE pdp_delete_root ( +CREATE TABLE pdp_root_delete ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -450,7 +454,7 @@ CREATE TABLE pdp_delete_root ( -- Main ProofSet Root table. Any and all root ever added by SP must be part of this table CREATE TABLE pdp_proofset_root ( - proofset BIGINT NOT NULL, -- pdp_proof_sets.id + proof_set_id BIGINT NOT NULL, -- pdp_proof_sets.id client TEXT NOT NULL, piece_cid_v2 TEXT NOT NULL, -- root cid (piececid v2) @@ -471,7 +475,7 @@ CREATE TABLE pdp_proofset_root ( remove_message_hash TEXT DEFAULT NULL, remove_message_index BIGINT DEFAULT NULL, - PRIMARY KEY (proofset, root) + PRIMARY KEY (proof_set_id, root) ); CREATE TABLE pdp_pipeline ( @@ -487,7 +491,7 @@ CREATE TABLE pdp_pipeline ( proof_set_id BIGINT NOT NULL, - extra_data BYTEA NOT NULL, + extra_data BYTEA, piece_ref BIGINT DEFAULT NULL, @@ -498,9 +502,6 @@ CREATE TABLE pdp_pipeline ( agg_task_id BIGINT DEFAULT NULL, aggregated BOOLEAN DEFAULT FALSE, - save_cache_task_id BIGINT DEFAULT NULL, - after_save_cache BOOLEAN DEFAULT FALSE, - add_root_task_id BIGINT DEFAULT NULL, after_add_root BOOLEAN DEFAULT FALSE, @@ -509,11 +510,16 @@ CREATE TABLE pdp_pipeline ( after_add_root_msg BOOLEAN DEFAULT FALSE, + save_cache_task_id BIGINT DEFAULT NULL, + after_save_cache BOOLEAN DEFAULT FALSE, + indexing BOOLEAN DEFAULT FALSE, indexing_created_at TIMESTAMPTZ DEFAULT NULL, indexing_task_id BIGINT DEFAULT NULL, indexed BOOLEAN DEFAULT FALSE, + announce BOOLEAN DEFAULT FALSE, + complete BOOLEAN DEFAULT FALSE, PRIMARY KEY (id, aggr_index) @@ -524,4 +530,66 @@ CREATE TABLE market_mk20_clients ( allowed BOOLEAN DEFAULT TRUE ); +-- IPNI pipeline is kept separate from rest for robustness +-- and reuse. This allows for removing, recreating ads using CLI. +CREATE TABLE pdp_ipni_task ( + context_id BYTEA NOT NULL, + is_rm BOOLEAN NOT NULL, + + id TEXT NOT NULL, + + provider TEXT NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), + task_id BIGINT DEFAULT NULL, + complete BOOLEAN DEFAULT FALSE, + + PRIMARY KEY (context_id, is_rm) +); + + +-- Function to create ipni tasks +CREATE OR REPLACE FUNCTION insert_pdp_ipni_task( + _context_id BYTEA, + _is_rm BOOLEAN, + _id TEXT, + _provider TEXT, + _task_id BIGINT DEFAULT NULL +) RETURNS VOID AS $$ +DECLARE +_existing_is_rm BOOLEAN; +_latest_is_rm BOOLEAN; +BEGIN + -- Check if ipni_task has the same context_id and provider with a different is_rm value + SELECT is_rm INTO _existing_is_rm + FROM ipni_task + WHERE provider = _provider AND context_id = _context_id AND is_rm != _is_rm + LIMIT 1; + + -- If a different is_rm exists for the same context_id and provider, insert the new task + IF FOUND THEN + INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id, created_at, complete) + VALUES (_context_id, _is_rm, _id, _provider, _task_id, TIMEZONE('UTC', NOW()), FALSE); + RETURN; + END IF; + + -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table + SELECT is_rm INTO _latest_is_rm + FROM ipni + WHERE provider = _provider AND context_id = _context_id + ORDER BY order_number DESC + LIMIT 1; + + -- If the latest ad has the same is_rm value, raise an exception + IF FOUND AND _latest_is_rm = _is_rm THEN + RAISE EXCEPTION 'already published'; + END IF; + + -- If all conditions are met, insert the new task into ipni_task + INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id, created_at, complete) + VALUES (_context_id, _is_rm, _id, _provider, _task_id, TIMEZONE('UTC', NOW()), FALSE); +END; +$$ LANGUAGE plpgsql; + + diff --git a/market/ipni/ipni-provider/ipni-provider.go b/market/ipni/ipni-provider/ipni-provider.go index 33478e266..f02995b47 100644 --- a/market/ipni/ipni-provider/ipni-provider.go +++ b/market/ipni/ipni-provider/ipni-provider.go @@ -103,8 +103,9 @@ func NewProvider(d *deps.Deps) (*Provider, error) { for rows.Next() && rows.Err() == nil { var priv []byte var peerID string + var sp int64 var spID abi.ActorID - err := rows.Scan(&priv, &peerID, &spID) + err := rows.Scan(&priv, &peerID, &sp) if err != nil { return nil, xerrors.Errorf("failed to scan the row: %w", err) } @@ -123,6 +124,10 @@ func NewProvider(d *deps.Deps) (*Provider, error) { return nil, xerrors.Errorf("peer ID mismatch: got %s (calculated), expected %s (DB)", id.String(), peerID) } + if sp < 0 { + spID = abi.ActorID(0) + } + maddr, err := address.NewIDAddress(uint64(spID)) if err != nil { return nil, xerrors.Errorf("parsing miner ID: %w", err) diff --git a/market/ipni/ipni-provider/spark.go b/market/ipni/ipni-provider/spark.go index eb661a292..e3b46d44a 100644 --- a/market/ipni/ipni-provider/spark.go +++ b/market/ipni/ipni-provider/spark.go @@ -27,6 +27,9 @@ import ( func (p *Provider) updateSparkContract(ctx context.Context) error { for _, pInfo := range p.keys { pInfo := pInfo + if pInfo.SPID == 0 { + return nil + } mInfo, err := p.full.StateMinerInfo(ctx, pInfo.Miner, types.EmptyTSK) if err != nil { return err diff --git a/market/mk20/client/auth.go b/market/mk20/client/auth.go new file mode 100644 index 000000000..fdc57984a --- /dev/null +++ b/market/mk20/client/auth.go @@ -0,0 +1,42 @@ +package client + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "fmt" + "time" +) + +// Signer abstracts the signature operation (ed25519, secp256k1, ā€¦). +type Signer interface { + // Sign signs the supplied digest and returns raw signature bytes. + Sign(digest []byte) ([]byte, error) + // PublicKeyBytes returns the raw publicā€‘key bytes (no multibase / address). + PublicKeyBytes() []byte + // Type returns a short string identifying the key algorithm ("ed25519", ā€¦). + Type() string +} + +// HourlyCurioAuthHeader returns a HTTPClient Option that injects ā€œCurioAuth ā€¦ā€ +// on every request using the algorithm defined in the OpenAPI spec. +func HourlyCurioAuthHeader(s Signer) Option { + return WithAuth(func(_ context.Context) (string, string, error) { + now := time.Now().UTC().Truncate(time.Hour) + msg := bytes.Join([][]byte{s.PublicKeyBytes(), []byte(now.Format(time.RFC3339))}, []byte{}) + digest := sha256.Sum256(msg) + + sig, err := s.Sign(digest[:]) + if err != nil { + return "", "", err + } + + header := fmt.Sprintf("CurioAuth %s:%s:%s", + s.Type(), + base64.StdEncoding.EncodeToString(s.PublicKeyBytes()), + base64.StdEncoding.EncodeToString(sig), + ) + return "Authorization", header, nil + }) +} diff --git a/market/mk20/client/client.go b/market/mk20/client/client.go new file mode 100644 index 000000000..975c22ad3 --- /dev/null +++ b/market/mk20/client/client.go @@ -0,0 +1,404 @@ +package client + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "github.com/oklog/ulid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" + + "github.com/filecoin-project/curio/market/mk20" +) + +var log = logging.Logger("mk20-client") + +type Client struct { + http *HTTPClient +} + +func NewClient(baseURL, auth string) *Client { + hclient := New(baseURL, Option(WithAuthString(auth))) + return &Client{ + http: hclient, + } +} + +func (c *Client) Deal(ctx context.Context, maddr, wallet address.Address, pieceCid cid.Cid, http_url, aggregateFile, contract_address, contract_method string, headers http.Header, put, index, announce, pdp bool, duration, allocation, proofSet int64) error { + var d mk20.DataSource + + if aggregateFile != "" { + d = mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Aggregate: &mk20.FormatAggregate{ + Type: mk20.AggregateTypeV1, + }, + }, + } + + var pieces []mk20.DataSource + + log.Debugw("using aggregate data source", "aggregate", aggregateFile) + // Read file line by line + loc, err := homedir.Expand(aggregateFile) + if err != nil { + return err + } + file, err := os.Open(loc) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Split(line, "\t") + if len(parts) != 2 { + return fmt.Errorf("invalid line format. Expected pieceCidV2, url at %s", line) + } + if parts[0] == "" || parts[1] == "" { + return fmt.Errorf("empty column value in the input file at %s", line) + } + + pieceCid, err := cid.Parse(parts[0]) + if err != nil { + return fmt.Errorf("failed to parse CID: %w", err) + } + + url, err := url.Parse(parts[1]) + if err != nil { + return fmt.Errorf("failed to parse url: %w", err) + } + + pieces = append(pieces, mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Priority: 0, + Fallback: true, + }, + }, + }, + }) + + if err := scanner.Err(); err != nil { + return err + } + } + d.SourceAggregate = &mk20.DataSourceAggregate{ + Pieces: pieces, + } + } else { + if http_url == "" { + if put { + d = mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHttpPut: &mk20.DataSourceHttpPut{}, + } + } else { + d = mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceOffline: &mk20.DataSourceOffline{}, + } + } + } else { + url, err := url.Parse(http_url) + if err != nil { + return xerrors.Errorf("parsing http url: %w", err) + } + d = mk20.DataSource{ + PieceCID: pieceCid, + Format: mk20.PieceDataFormat{ + Car: &mk20.FormatCar{}, + }, + SourceHTTP: &mk20.DataSourceHTTP{ + URLs: []mk20.HttpUrl{ + { + URL: url.String(), + Headers: headers, + Priority: 0, + Fallback: true, + }, + }, + }, + } + } + } + + p := mk20.Products{ + DDOV1: &mk20.DDOV1{ + Provider: maddr, + PieceManager: wallet, + Duration: abi.ChainEpoch(duration), + ContractAddress: contract_address, + ContractVerifyMethod: contract_method, + ContractVerifyMethodParams: []byte("test bytes"), + }, + RetrievalV1: &mk20.RetrievalV1{ + Indexing: index, + AnnouncePayload: announce, + }, + } + + if pdp { + ps := uint64(proofSet) + p.PDPV1 = &mk20.PDPV1{ + AddRoot: true, + ProofSetID: &ps, + ExtraData: []byte("test bytes"), // TODO: Fix this + } + } + + if allocation != 0 { + alloc := verifreg.AllocationId(allocation) + p.DDOV1.AllocationId = &alloc + } + + id, err := mk20.NewULID() + if err != nil { + return err + } + log.Debugw("generated deal id", "id", id) + + deal := mk20.Deal{ + Identifier: id, + Client: wallet, + Data: &d, + Products: p, + } + + log.Debugw("deal", "deal", deal) + + rerr := c.http.Store(ctx, &deal) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealStatus(ctx context.Context, dealID string) (*mk20.DealProductStatusResponse, error) { + id, err := ulid.Parse(dealID) + if err != nil { + return nil, xerrors.Errorf("parsing deal id: %w", err) + } + + status, rerr := c.http.Status(ctx, id) + if rerr.Error != nil { + return nil, rerr.Error + } + if rerr.Status != 200 { + return nil, rerr.HError() + } + + return status, nil +} + +func (c *Client) DealUpdate(ctx context.Context, dealID string, deal *mk20.Deal) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + rerr := c.http.Update(ctx, id, deal) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealUploadSerial(ctx context.Context, dealID string, r io.Reader) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + rerr := c.http.UploadSerial(ctx, id, r) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealUploadSerialFinalize(ctx context.Context, dealID string, deal *mk20.Deal) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + rerr := c.http.UploadSerialFinalize(ctx, id, deal) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealChunkUploadInit(ctx context.Context, dealID string, fileSize, chunkSize int64) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + metadata := &mk20.StartUpload{ + RawSize: uint64(fileSize), + ChunkSize: chunkSize, + } + rerr := c.http.UploadInit(ctx, id, metadata) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealChunkUpload(ctx context.Context, dealID string, chunk int, r io.Reader) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + rerr := c.http.UploadChunk(ctx, id, chunk, r) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealChunkUploadFinalize(ctx context.Context, dealID string, deal *mk20.Deal) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + rerr := c.http.UploadSerialFinalize(ctx, id, deal) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} + +func (c *Client) DealChunkedUpload(ctx context.Context, dealID string, size, chunkSize int64, r io.ReaderAt) error { + id, err := ulid.Parse(dealID) + if err != nil { + return xerrors.Errorf("parsing deal id: %w", err) + } + metadata := &mk20.StartUpload{ + RawSize: uint64(size), + ChunkSize: chunkSize, + } + + _, rerr := c.http.UploadStatus(ctx, id) + if rerr.Error != nil { + return rerr.Error + } + + if rerr.Status != 200 && rerr.Status != int(mk20.UploadStatusCodeUploadNotStarted) { + return rerr.HError() + } + + if rerr.Status == int(mk20.UploadStatusCodeUploadNotStarted) { + // Start the upload + rerr = c.http.UploadInit(ctx, id, metadata) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + } + + numChunks := int(size / chunkSize) + + for { + status, rerr := c.http.UploadStatus(ctx, id) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + + log.Debugw("upload status", "status", status) + + if status.TotalChunks != numChunks { + return xerrors.Errorf("expected %d chunks, got %d", numChunks, status.TotalChunks) + } + + if status.Missing == 0 { + break + } + + log.Warnw("missing chunks", "missing", status.Missing) + // Try to upload missing chunks + for _, chunk := range status.MissingChunks { + start := int64(chunk-1) * chunkSize + end := start + chunkSize + if end > size { + end = size + } + log.Debugw("uploading chunk", "start", start, "end", end) + buf := make([]byte, end-start) + _, err := r.ReadAt(buf, start) + if err != nil { + return xerrors.Errorf("failed to read chunk: %w", err) + } + + rerr = c.http.UploadChunk(ctx, id, chunk, bytes.NewReader(buf)) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + } + } + + log.Infow("upload complete") + + rerr = c.http.UploadFinalize(ctx, id, nil) + if rerr.Error != nil { + return rerr.Error + } + if rerr.Status != 200 { + return rerr.HError() + } + return nil +} diff --git a/market/mk20/client/http_client.go b/market/mk20/client/http_client.go new file mode 100644 index 000000000..b44cfbb56 --- /dev/null +++ b/market/mk20/client/http_client.go @@ -0,0 +1,235 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "strconv" + "time" + + "github.com/oklog/ulid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/market/mk20" +) + +const MarketPath = "/market/mk20" + +// HTTPClient is a thin wrapper around Curio Market 2.0 REST endpoints. +type HTTPClient struct { + BaseURL string + HTTP *http.Client + AuthHeader func(context.Context) (key string, value string, err error) + AuthHeaderString string +} + +// New returns a HTTPClient with sane defaults. +func New(baseURL string, opts ...Option) *HTTPClient { + c := &HTTPClient{ + BaseURL: baseURL + MarketPath, + HTTP: &http.Client{Timeout: 60 * time.Second}, + } + for _, o := range opts { + o(c) + } + return c +} + +// --- options --------------------------------------------------------------- + +type Option func(*HTTPClient) + +func WithAuth(h func(context.Context) (string, string, error)) Option { + return func(c *HTTPClient) { c.AuthHeader = h } +} + +func WithAuthString(s string) Option { + return func(c *HTTPClient) { c.AuthHeaderString = s } +} + +// --- lowā€‘level helper ------------------------------------------------------ + +func (c *HTTPClient) do(ctx context.Context, method, p string, body io.Reader, v any) *Error { + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path.Clean("/"+p), body) + if err != nil { + return &Error{ + Status: 0, + Error: err, + } + } + + if c.AuthHeaderString != "" { + if c.AuthHeader != nil { + k, vHdr, err := c.AuthHeader(ctx) + if err != nil { + return &Error{ + Status: 0, + Error: err, + } + } + req.Header.Set(k, vHdr) + } + } else { + req.Header.Set("Authorization", c.AuthHeaderString) + } + + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.HTTP.Do(req) + if err != nil { + return &Error{ + Status: 0, + Error: err, + } + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + msg, err := io.ReadAll(resp.Body) + if err != nil { + return &Error{Status: resp.StatusCode, Error: err} + } + return &Error{Status: resp.StatusCode, Message: string(msg)} + } + + if v != nil { + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return &Error{Status: resp.StatusCode, Error: err} + } + } + return nil +} + +// Error wraps nonā€‘2xx responses. +type Error struct { + Status int + Message string + Error error +} + +func (e *Error) HError() error { + return xerrors.Errorf("%s", fmt.Sprintf("curio: %d ā€“ %s", e.Status, e.Message)) +} + +// --- public methods (one per path) ---------------------------------------- + +// /contracts +func (c *HTTPClient) Contracts(ctx context.Context) ([]string, *Error) { + var out []string + err := c.do(ctx, http.MethodGet, "/contracts", nil, &out) + return out, err +} + +// /products +func (c *HTTPClient) Products(ctx context.Context) ([]string, *Error) { + var out []string + err := c.do(ctx, http.MethodGet, "/products", nil, &out) + return out, err +} + +// /sources +func (c *HTTPClient) Sources(ctx context.Context) ([]string, *Error) { + var out []string + err := c.do(ctx, http.MethodGet, "/sources", nil, &out) + return out, err +} + +// /status/{id} +func (c *HTTPClient) Status(ctx context.Context, id ulid.ULID) (*mk20.DealProductStatusResponse, *Error) { + var out mk20.DealProductStatusResponse + err := c.do(ctx, http.MethodGet, "/status/"+id.String(), nil, &out) + return &out, err +} + +// /store (POST) +func (c *HTTPClient) Store(ctx context.Context, deal *mk20.Deal) *Error { + b, merr := json.Marshal(deal) + if merr != nil { + return &Error{Status: 0, Error: xerrors.Errorf("failed to marshal deal: %w", merr)} + } + err := c.do(ctx, http.MethodPost, "/store", bytes.NewReader(b), nil) + return err +} + +// /update/{id} (GET in spec ā€“ unusual, but honoured) +func (c *HTTPClient) Update(ctx context.Context, id ulid.ULID, deal *mk20.Deal) *Error { + b, merr := json.Marshal(deal) + if merr != nil { + return &Error{Status: 0, Error: xerrors.Errorf("failed to marshal deal: %w", merr)} + } + err := c.do(ctx, http.MethodGet, "/update/"+id.String(), bytes.NewReader(b), nil) + return err +} + +// Serial upload (small files) ā€“ PUT /upload/{id} +func (c *HTTPClient) UploadSerial(ctx context.Context, id ulid.ULID, r io.Reader) *Error { + err := c.do(ctx, http.MethodPut, "/upload/"+id.String(), r, nil) + return err +} + +// Finalize serial upload ā€“ POST /upload/{id} +func (c *HTTPClient) UploadSerialFinalize(ctx context.Context, id ulid.ULID, deal *mk20.Deal) *Error { + var err *Error + if deal != nil { + b, merr := json.Marshal(deal) + if merr != nil { + return &Error{Status: 0, Error: xerrors.Errorf("failed to marshal deal: %w", merr)} + } + err = c.do(ctx, http.MethodPost, "/upload/"+id.String(), bytes.NewReader(b), nil) + } else { + err = c.do(ctx, http.MethodPost, "/upload/"+id.String(), nil, nil) + } + + return err +} + +// Chunked upload workflow --------------------------------------------------- + +// POST /uploads/{id} +func (c *HTTPClient) UploadInit(ctx context.Context, id ulid.ULID, metadata *mk20.StartUpload) *Error { + if metadata == nil { + return &Error{Status: 0, Error: xerrors.Errorf("metadata is required")} + } + b, merr := json.Marshal(metadata) + if merr != nil { + return &Error{Status: 0, Error: xerrors.Errorf("failed to marshal deal: %w", merr)} + } + err := c.do(ctx, http.MethodPost, "/uploads/"+id.String(), bytes.NewReader(b), nil) + return err +} + +// PUT /uploads/{id}/{chunk} +func (c *HTTPClient) UploadChunk(ctx context.Context, id ulid.ULID, chunk int, data io.Reader) *Error { + path := "/uploads/" + id.String() + "/" + strconv.Itoa(chunk) + err := c.do(ctx, http.MethodPut, path, data, nil) + return err +} + +// GET /uploads/{id} +func (c *HTTPClient) UploadStatus(ctx context.Context, id ulid.ULID) (*mk20.UploadStatus, *Error) { + var out mk20.UploadStatus + err := c.do(ctx, http.MethodGet, "/uploads/"+id.String(), nil, &out) + return &out, err +} + +// POST /uploads/finalize/{id} +func (c *HTTPClient) UploadFinalize(ctx context.Context, id ulid.ULID, deal *mk20.Deal) *Error { + var err *Error + if deal != nil { + b, merr := json.Marshal(deal) + if merr != nil { + return &Error{Status: 0, Error: xerrors.Errorf("failed to marshal deal: %w", merr)} + } + err = c.do(ctx, http.MethodPost, "/uploads/finalize/"+id.String(), bytes.NewReader(b), nil) + } else { + err = c.do(ctx, http.MethodPost, "/uploads/finalize/"+id.String(), nil, nil) + } + return err +} diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index 2db6249b1..3993214cd 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -41,7 +41,7 @@ type DDOV1 struct { Duration abi.ChainEpoch `json:"duration"` // AllocationId represents an aggregated allocation identifier for the deal. - AllocationId *verifreg.AllocationId `json:"allocation_id"` + AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty"` // ContractAddress specifies the address of the contract governing the deal ContractAddress string `json:"contract_address"` @@ -50,13 +50,13 @@ type DDOV1 struct { ContractVerifyMethod string `json:"contract_verify_method"` // ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - ContractVerifyMethodParams []byte `json:"contract_verify_method_params"` + ContractVerifyMethodParams []byte `json:"contract_verify_method_params,omitempty"` // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated NotificationAddress string `json:"notification_address"` // NotificationPayload holds the notification data typically in a serialized byte array format. - NotificationPayload []byte `json:"notification_payload"` + NotificationPayload []byte `json:"notification_payload,omitempty"` } func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { @@ -117,27 +117,27 @@ func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return Ok, nil } -func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (string, DealCode, error) { +func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient.Client) (int64, DealCode, error) { if d.ContractAddress == "0xtest" { v, err := rand.Int(rand.Reader, big.NewInt(10000000)) if err != nil { - return "", ErrServerInternalError, xerrors.Errorf("failed to generate random number: %w", err) + return -1, ErrServerInternalError, xerrors.Errorf("failed to generate random number: %w", err) } - return v.String(), Ok, nil + return v.Int64(), Ok, nil } var abiStr string err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return "", ErrMarketNotEnabled, UnknowContract + return -1, ErrMarketNotEnabled, UnknowContract } - return "", ErrServerInternalError, xerrors.Errorf("getting abi: %w", err) + return -1, ErrServerInternalError, xerrors.Errorf("getting abi: %w", err) } parsedABI, err := eabi.JSON(strings.NewReader(abiStr)) if err != nil { - return "", ErrServerInternalError, xerrors.Errorf("parsing abi: %w", err) + return -1, ErrServerInternalError, xerrors.Errorf("parsing abi: %w", err) } to := common.HexToAddress(d.ContractAddress) @@ -145,18 +145,18 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Get the method method, exists := parsedABI.Methods[d.ContractVerifyMethod] if !exists { - return "", ErrServerInternalError, fmt.Errorf("method %s not found in ABI", d.ContractVerifyMethod) + return -1, ErrServerInternalError, fmt.Errorf("method %s not found in ABI", d.ContractVerifyMethod) } // Enforce method must take exactly one `bytes` parameter if len(method.Inputs) != 1 || method.Inputs[0].Type.String() != "bytes" { - return "", ErrServerInternalError, fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) + return -1, ErrServerInternalError, fmt.Errorf("method %q must take exactly one argument of type bytes", method.Name) } // ABI-encode method call with input callData, err := parsedABI.Pack(method.Name, d.ContractVerifyMethod) if err != nil { - return "", ErrServerInternalError, fmt.Errorf("failed to encode call data: %w", err) + return -1, ErrServerInternalError, fmt.Errorf("failed to encode call data: %w", err) } // Build call message @@ -168,17 +168,17 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. // Call contract output, err := eth.CallContract(ctx, msg, nil) if err != nil { - return "", ErrServerInternalError, fmt.Errorf("eth_call failed: %w", err) + return -1, ErrServerInternalError, fmt.Errorf("eth_call failed: %w", err) } // Decode return value (assume string) - var result string + var result int64 if err := parsedABI.UnpackIntoInterface(&result, method.Name, output); err != nil { - return "", ErrServerInternalError, fmt.Errorf("decode result: %w", err) + return -1, ErrServerInternalError, fmt.Errorf("decode result: %w", err) } - if result == "" { - return "", ErrDealRejectedByMarket, fmt.Errorf("empty result from contract") + if result == 0 { + return -1, ErrDealRejectedByMarket, fmt.Errorf("empty result from contract") } return result, Ok, nil diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 7d3fd6955..340aaeeec 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -21,9 +21,9 @@ const docTemplate = `{ "summary": "List of supported DDO contracts", "responses": { "200": { - "description": "OK - Success", + "description": "Array of contract addresses supported by a system or application.", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedContracts" } }, "500": { @@ -41,9 +41,9 @@ const docTemplate = `{ "summary": "List of supported products", "responses": { "200": { - "description": "OK - Success", + "description": "Array of products supported by the SP", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedProducts" } }, "500": { @@ -61,9 +61,9 @@ const docTemplate = `{ "summary": "List of supported dats sources", "responses": { "200": { - "description": "OK - Success", + "description": "Array of dats sources supported by the SP", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedDataSources" } }, "500": { @@ -90,9 +90,9 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "OK - Success", + "description": "the status response for deal products with their respective deal statuses", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.DealProductStatusResponse" } }, "400": { @@ -619,9 +619,9 @@ const docTemplate = `{ ], "responses": { "200": { - "description": "UploadStatusCodeOk represents a successful upload operation with status code 200", + "description": "The status of a file upload process, including progress and missing chunks", "schema": { - "$ref": "#/definitions/mk20.UploadStatusCode" + "$ref": "#/definitions/mk20.UploadStatus" } }, "400": { @@ -652,6 +652,9 @@ const docTemplate = `{ }, "post": { "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", + "consumes": [ + "application/json" + ], "summary": "Starts the upload process", "parameters": [ { @@ -660,6 +663,15 @@ const docTemplate = `{ "name": "id", "in": "path", "required": true + }, + { + "description": "Metadata for initiating an upload operation", + "name": "data", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.StartUpload" + } } ], "responses": { @@ -717,7 +729,7 @@ const docTemplate = `{ }, { "description": "raw binary", - "name": "body", + "name": "data", "in": "body", "required": true, "schema": { @@ -1013,6 +1025,65 @@ const docTemplate = `{ "ErrDurationTooShort" ] }, + "mk20.DealProductStatusResponse": { + "type": "object", + "properties": { + "ddo_v1": { + "description": "DDOV1 holds the DealStatusResponse for product \"ddo_v1\".", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealStatusResponse" + } + ] + }, + "pdp_v1": { + "description": "PDPV1 represents the DealStatusResponse for the product pdp_v1.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealStatusResponse" + } + ] + } + } + }, + "mk20.DealState": { + "type": "string", + "enum": [ + "accepted", + "uploading", + "processing", + "sealing", + "indexing", + "failed", + "complete" + ], + "x-enum-varnames": [ + "DealStateAccepted", + "DealStateAwaitingUpload", + "DealStateProcessing", + "DealStateSealing", + "DealStateIndexing", + "DealStateFailed", + "DealStateComplete" + ] + }, + "mk20.DealStatusResponse": { + "type": "object", + "properties": { + "error_msg": { + "description": "ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.", + "type": "string" + }, + "status": { + "description": "State indicates the current processing state of the deal as a DealState value.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealState" + } + ] + } + } + }, "mk20.FormatAggregate": { "type": "object", "properties": { @@ -1182,6 +1253,55 @@ const docTemplate = `{ } } }, + "mk20.StartUpload": { + "type": "object", + "properties": { + "chunk_size": { + "description": "ChunkSize defines the size of each data chunk to be used during the upload process.", + "type": "integer" + }, + "raw_size": { + "description": "RawSize indicates the total size of the data to be uploaded in bytes.", + "type": "integer" + } + } + }, + "mk20.SupportedContracts": { + "type": "object", + "properties": { + "contracts": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "mk20.SupportedDataSources": { + "type": "object", + "properties": { + "sources": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "mk20.SupportedProducts": { + "type": "object", + "properties": { + "products": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "mk20.UploadCode": { "type": "integer", "enum": [ @@ -1216,6 +1336,37 @@ const docTemplate = `{ "UploadStartCodeServerError" ] }, + "mk20.UploadStatus": { + "type": "object", + "properties": { + "missing": { + "description": "Missing represents the number of chunks that are not yet uploaded.", + "type": "integer" + }, + "missing_chunks": { + "description": "MissingChunks is a slice containing the indices of missing chunks.", + "type": "array", + "items": { + "type": "integer" + } + }, + "total_chunks": { + "description": "TotalChunks represents the total number of chunks required for the upload.", + "type": "integer" + }, + "uploaded": { + "description": "Uploaded represents the number of chunks successfully uploaded.", + "type": "integer" + }, + "uploaded_chunks": { + "description": "UploadedChunks is a slice containing the indices of successfully uploaded chunks.", + "type": "array", + "items": { + "type": "integer" + } + } + } + }, "mk20.UploadStatusCode": { "type": "integer", "enum": [ @@ -1234,7 +1385,7 @@ const docTemplate = `{ }, "securityDefinitions": { "CurioAuth": { - "description": "Use the format: ` + "`" + `CurioAuth PublicKeyType:PublicKey:Signature` + "`" + `\n\n- ` + "`" + `PublicKeyType` + "`" + `: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- ` + "`" + `PublicKey` + "`" + `: Base64 string of public key bytes\n- ` + "`" + `Signature` + "`" + `: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order.\n- The raw public key bytes (not a human-readable address)\n- The HTTP request path, such as /user/info\n- The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z)\n- These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", + "description": "Use the format: ` + "`" + `CurioAuth PublicKeyType:PublicKey:Signature` + "`" + `\n\n- ` + "`" + `PublicKeyType` + "`" + `: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- ` + "`" + `PublicKey` + "`" + `: Base64 string of public key bytes\n- ` + "`" + `Signature` + "`" + `: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following components, in order.\n- The raw public key bytes (not a human-readable address)\n- The timestamp, truncated to the nearest hour, formatted in RFC3339 (e.g., 2025-07-15T17:00:00Z)\n- These two byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 0dd16e416..d4fe1bbe5 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -9,6 +9,8 @@ import ( "fmt" "io" "net/http" + "runtime" + "runtime/debug" "strconv" "time" @@ -22,6 +24,7 @@ import ( "github.com/filecoin-project/go-address" + "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/market/mk20" @@ -63,13 +66,18 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { func AuthMiddleware(db *harmonydb.DB) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // TODO: Remove this check once Synapse integration is done + if build.BuildType != build.BuildMainnet { + next.ServeHTTP(w, r) + return + } authHeader := r.Header.Get("Authorization") if authHeader == "" { http.Error(w, "Missing Authorization header", http.StatusUnauthorized) return } - allowed, client, err := mk20.Auth(authHeader, r.URL.Path, db) + allowed, client, err := mk20.Auth(authHeader, db) if err != nil { log.Errorw("failed to authenticate request", "err", err) http.Error(w, err.Error(), http.StatusUnauthorized) @@ -118,11 +126,10 @@ func Router(mdh *MK20DealHandler, domainName string) http.Handler { // @description - `PublicKeyType`: String representation of type of wallet (e.g., "ed25519", "bls", "secp256k1") // @description - `PublicKey`: Base64 string of public key bytes // @description - `Signature`: Signature is Base64 string of signature bytes. -// @description - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order. +// @description - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following components, in order. // @description - The raw public key bytes (not a human-readable address) -// @description - The HTTP request path, such as /user/info -// @description - The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z) -// @description - These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. +// @description - The timestamp, truncated to the nearest hour, formatted in RFC3339 (e.g., 2025-07-15T17:00:00Z) +// @description - These two byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. // @security CurioAuth func APIRouter(mdh *MK20DealHandler, domainName string) http.Handler { SwaggerInfo.BasePath = "/market/mk20" @@ -198,6 +205,15 @@ func InfoRouter() http.Handler { // @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + log.Errorf("panic occurred: %v\n%s", r, trace[:n]) + debug.PrintStack() + } + }() + ct := r.Header.Get("Content-Type") var deal mk20.Deal if ct != "application/json" { @@ -223,6 +239,8 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { return } + log.Infof("DATA IS NULL = %t\n", deal.Data == nil) + log.Infow("received deal proposal", "deal", deal) result := mdh.dm.MK20Handler.ExecuteDeal(context.Background(), &deal) @@ -245,8 +263,8 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { // @Description List of supported DDO contracts // @BasePath /market/mk20 // @Param id path string true "id" +// @Failure 200 {object} mk20.DealProductStatusResponse "the status response for deal products with their respective deal statuses" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" -// @Failure 200 {string} string "OK - Success" // @Failure 500 {string} string "Internal Server Error" func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") @@ -288,8 +306,8 @@ func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { // @Summary List of supported DDO contracts // @Description List of supported DDO contracts // @BasePath /market/mk20 +// @Failure 200 {object} mk20.SupportedContracts "Array of contract addresses supported by a system or application." // @Failure 500 {string} string "Internal Server Error" -// @Failure 200 {string} string "OK - Success" func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *http.Request) { var contracts mk20.SupportedContracts err := mdh.db.Select(r.Context(), &contracts, "SELECT address FROM ddo_contracts") @@ -324,7 +342,7 @@ func (mdh *MK20DealHandler) mk20supportedContracts(w http.ResponseWriter, r *htt // @Description List of supported products // @BasePath /market/mk20 // @Failure 500 {string} string "Internal Server Error" -// @Failure 200 {string} string "OK - Success" +// @Failure 200 {object} mk20.SupportedProducts "Array of products supported by the SP" func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Request) { prods, _, err := mdh.dm.MK20Handler.Supported(r.Context()) if err != nil { @@ -358,7 +376,7 @@ func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Req // @Description List of supported data sources // @BasePath /market/mk20 // @Failure 500 {string} string "Internal Server Error" -// @Failure 200 {string} string "OK - Success" +// @Failure 200 {object} mk20.SupportedDataSources "Array of dats sources supported by the SP" func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http.Request) { _, srcs, err := mdh.dm.MK20Handler.Supported(r.Context()) if err != nil { @@ -392,7 +410,7 @@ func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http. // @Summary Status of deal upload // @Description Return a json struct detailing the current status of a deal upload. // @BasePath /market/mk20 -// @Failure 200 {object} mk20.UploadStatusCode "UploadStatusCodeOk represents a successful upload operation with status code 200" +// @Failure 200 {object} mk20.UploadStatus "The status of a file upload process, including progress and missing chunks" // @Failure 404 {object} mk20.UploadStatusCode "UploadStatusCodeDealNotFound indicates that the requested deal was not found, corresponding to status code 404" // @Failure 425 {object} mk20.UploadStatusCode "UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet" // @Failure 500 {object} mk20.UploadStatusCode "UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500" @@ -421,7 +439,7 @@ func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Requ // @Param id path string true "id" // @Param chunkNum path string true "chunkNum" // @accepts bytes -// @Param body body []byte true "raw binary" +// @Param data body []byte true "raw binary" // @Failure 200 {object} mk20.UploadCode "UploadOk indicates a successful upload operation, represented by the HTTP status code 200" // @Failure 400 {object} mk20.UploadCode "UploadBadRequest represents a bad request error with an HTTP status code of 400" // @Failure 404 {object} mk20.UploadCode "UploadNotFound represents an error where the requested upload chunk could not be found, typically corresponding to HTTP status 404" @@ -471,7 +489,9 @@ func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http. // @Summary Starts the upload process // @Description Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal. // @BasePath /market/mk20 +// @Accept json // @Param id path string true "id" +// @Param data body mk20.StartUpload true "Metadata for initiating an upload operation" // @Failure 200 {object} mk20.UploadStartCode "UploadStartCodeOk indicates a successful upload start request with status code 200" // @Failure 400 {object} mk20.UploadStartCode "UploadStartCodeBadRequest indicates a bad upload start request error with status code 400" // @Failure 404 {object} mk20.UploadStartCode "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process" @@ -741,7 +761,7 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h } ct := r.Header.Get("Content-Type") - // If Content-Type is not set this is does not require updating the deal + // If Content-Type is not set, it is not required to update the deal if len(ct) == 0 { log.Infow("received finalize upload proposal without content type", "id", id) mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, nil, w) diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index a8740391d..d14c9d2bc 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -12,9 +12,9 @@ "summary": "List of supported DDO contracts", "responses": { "200": { - "description": "OK - Success", + "description": "Array of contract addresses supported by a system or application.", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedContracts" } }, "500": { @@ -32,9 +32,9 @@ "summary": "List of supported products", "responses": { "200": { - "description": "OK - Success", + "description": "Array of products supported by the SP", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedProducts" } }, "500": { @@ -52,9 +52,9 @@ "summary": "List of supported dats sources", "responses": { "200": { - "description": "OK - Success", + "description": "Array of dats sources supported by the SP", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.SupportedDataSources" } }, "500": { @@ -81,9 +81,9 @@ ], "responses": { "200": { - "description": "OK - Success", + "description": "the status response for deal products with their respective deal statuses", "schema": { - "type": "string" + "$ref": "#/definitions/mk20.DealProductStatusResponse" } }, "400": { @@ -610,9 +610,9 @@ ], "responses": { "200": { - "description": "UploadStatusCodeOk represents a successful upload operation with status code 200", + "description": "The status of a file upload process, including progress and missing chunks", "schema": { - "$ref": "#/definitions/mk20.UploadStatusCode" + "$ref": "#/definitions/mk20.UploadStatus" } }, "400": { @@ -643,6 +643,9 @@ }, "post": { "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", + "consumes": [ + "application/json" + ], "summary": "Starts the upload process", "parameters": [ { @@ -651,6 +654,15 @@ "name": "id", "in": "path", "required": true + }, + { + "description": "Metadata for initiating an upload operation", + "name": "data", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/mk20.StartUpload" + } } ], "responses": { @@ -708,7 +720,7 @@ }, { "description": "raw binary", - "name": "body", + "name": "data", "in": "body", "required": true, "schema": { @@ -1004,6 +1016,65 @@ "ErrDurationTooShort" ] }, + "mk20.DealProductStatusResponse": { + "type": "object", + "properties": { + "ddo_v1": { + "description": "DDOV1 holds the DealStatusResponse for product \"ddo_v1\".", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealStatusResponse" + } + ] + }, + "pdp_v1": { + "description": "PDPV1 represents the DealStatusResponse for the product pdp_v1.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealStatusResponse" + } + ] + } + } + }, + "mk20.DealState": { + "type": "string", + "enum": [ + "accepted", + "uploading", + "processing", + "sealing", + "indexing", + "failed", + "complete" + ], + "x-enum-varnames": [ + "DealStateAccepted", + "DealStateAwaitingUpload", + "DealStateProcessing", + "DealStateSealing", + "DealStateIndexing", + "DealStateFailed", + "DealStateComplete" + ] + }, + "mk20.DealStatusResponse": { + "type": "object", + "properties": { + "error_msg": { + "description": "ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.", + "type": "string" + }, + "status": { + "description": "State indicates the current processing state of the deal as a DealState value.", + "allOf": [ + { + "$ref": "#/definitions/mk20.DealState" + } + ] + } + } + }, "mk20.FormatAggregate": { "type": "object", "properties": { @@ -1173,6 +1244,55 @@ } } }, + "mk20.StartUpload": { + "type": "object", + "properties": { + "chunk_size": { + "description": "ChunkSize defines the size of each data chunk to be used during the upload process.", + "type": "integer" + }, + "raw_size": { + "description": "RawSize indicates the total size of the data to be uploaded in bytes.", + "type": "integer" + } + } + }, + "mk20.SupportedContracts": { + "type": "object", + "properties": { + "contracts": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "mk20.SupportedDataSources": { + "type": "object", + "properties": { + "sources": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "mk20.SupportedProducts": { + "type": "object", + "properties": { + "products": { + "description": "Contracts represents a list of supported contract addresses in string format.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "mk20.UploadCode": { "type": "integer", "enum": [ @@ -1207,6 +1327,37 @@ "UploadStartCodeServerError" ] }, + "mk20.UploadStatus": { + "type": "object", + "properties": { + "missing": { + "description": "Missing represents the number of chunks that are not yet uploaded.", + "type": "integer" + }, + "missing_chunks": { + "description": "MissingChunks is a slice containing the indices of missing chunks.", + "type": "array", + "items": { + "type": "integer" + } + }, + "total_chunks": { + "description": "TotalChunks represents the total number of chunks required for the upload.", + "type": "integer" + }, + "uploaded": { + "description": "Uploaded represents the number of chunks successfully uploaded.", + "type": "integer" + }, + "uploaded_chunks": { + "description": "UploadedChunks is a slice containing the indices of successfully uploaded chunks.", + "type": "array", + "items": { + "type": "integer" + } + } + } + }, "mk20.UploadStatusCode": { "type": "integer", "enum": [ @@ -1225,7 +1376,7 @@ }, "securityDefinitions": { "CurioAuth": { - "description": "Use the format: `CurioAuth PublicKeyType:PublicKey:Signature`\n\n- `PublicKeyType`: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- `PublicKey`: Base64 string of public key bytes\n- `Signature`: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order.\n- The raw public key bytes (not a human-readable address)\n- The HTTP request path, such as /user/info\n- The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z)\n- These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", + "description": "Use the format: `CurioAuth PublicKeyType:PublicKey:Signature`\n\n- `PublicKeyType`: String representation of type of wallet (e.g., \"ed25519\", \"bls\", \"secp256k1\")\n- `PublicKey`: Base64 string of public key bytes\n- `Signature`: Signature is Base64 string of signature bytes.\n- The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following components, in order.\n- The raw public key bytes (not a human-readable address)\n- The timestamp, truncated to the nearest hour, formatted in RFC3339 (e.g., 2025-07-15T17:00:00Z)\n- These two byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash.", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index dfb732bc8..2d4e41728 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -177,6 +177,47 @@ definitions: - ErrServiceOverloaded - ErrMarketNotEnabled - ErrDurationTooShort + mk20.DealProductStatusResponse: + properties: + ddo_v1: + allOf: + - $ref: '#/definitions/mk20.DealStatusResponse' + description: DDOV1 holds the DealStatusResponse for product "ddo_v1". + pdp_v1: + allOf: + - $ref: '#/definitions/mk20.DealStatusResponse' + description: PDPV1 represents the DealStatusResponse for the product pdp_v1. + type: object + mk20.DealState: + enum: + - accepted + - uploading + - processing + - sealing + - indexing + - failed + - complete + type: string + x-enum-varnames: + - DealStateAccepted + - DealStateAwaitingUpload + - DealStateProcessing + - DealStateSealing + - DealStateIndexing + - DealStateFailed + - DealStateComplete + mk20.DealStatusResponse: + properties: + error_msg: + description: ErrorMsg is an optional field containing error details associated + with the deal's current state if an error occurred. + type: string + status: + allOf: + - $ref: '#/definitions/mk20.DealState' + description: State indicates the current processing state of the deal as a + DealState value. + type: object mk20.FormatAggregate: properties: sub: @@ -304,6 +345,44 @@ definitions: system to support CIDs based retrieval type: boolean type: object + mk20.StartUpload: + properties: + chunk_size: + description: ChunkSize defines the size of each data chunk to be used during + the upload process. + type: integer + raw_size: + description: RawSize indicates the total size of the data to be uploaded in + bytes. + type: integer + type: object + mk20.SupportedContracts: + properties: + contracts: + description: Contracts represents a list of supported contract addresses in + string format. + items: + type: string + type: array + type: object + mk20.SupportedDataSources: + properties: + sources: + description: Contracts represents a list of supported contract addresses in + string format. + items: + type: string + type: array + type: object + mk20.SupportedProducts: + properties: + products: + description: Contracts represents a list of supported contract addresses in + string format. + items: + type: string + type: array + type: object mk20.UploadCode: enum: - 200 @@ -332,6 +411,30 @@ definitions: - UploadStartCodeDealNotFound - UploadStartCodeAlreadyStarted - UploadStartCodeServerError + mk20.UploadStatus: + properties: + missing: + description: Missing represents the number of chunks that are not yet uploaded. + type: integer + missing_chunks: + description: MissingChunks is a slice containing the indices of missing chunks. + items: + type: integer + type: array + total_chunks: + description: TotalChunks represents the total number of chunks required for + the upload. + type: integer + uploaded: + description: Uploaded represents the number of chunks successfully uploaded. + type: integer + uploaded_chunks: + description: UploadedChunks is a slice containing the indices of successfully + uploaded chunks. + items: + type: integer + type: array + type: object mk20.UploadStatusCode: enum: - 200 @@ -354,9 +457,9 @@ paths: description: List of supported DDO contracts responses: "200": - description: OK - Success + description: Array of contract addresses supported by a system or application. schema: - type: string + $ref: '#/definitions/mk20.SupportedContracts' "500": description: Internal Server Error schema: @@ -367,9 +470,9 @@ paths: description: List of supported products responses: "200": - description: OK - Success + description: Array of products supported by the SP schema: - type: string + $ref: '#/definitions/mk20.SupportedProducts' "500": description: Internal Server Error schema: @@ -380,9 +483,9 @@ paths: description: List of supported data sources responses: "200": - description: OK - Success + description: Array of dats sources supported by the SP schema: - type: string + $ref: '#/definitions/mk20.SupportedDataSources' "500": description: Internal Server Error schema: @@ -399,9 +502,10 @@ paths: type: string responses: "200": - description: OK - Success + description: the status response for deal products with their respective + deal statuses schema: - type: string + $ref: '#/definitions/mk20.DealProductStatusResponse' "400": description: Bad Request - Invalid input or validation error schema: @@ -719,10 +823,10 @@ paths: type: string responses: "200": - description: UploadStatusCodeOk represents a successful upload operation - with status code 200 + description: The status of a file upload process, including progress and + missing chunks schema: - $ref: '#/definitions/mk20.UploadStatusCode' + $ref: '#/definitions/mk20.UploadStatus' "400": description: Bad Request - Invalid input or validation error schema: @@ -744,6 +848,8 @@ paths: $ref: '#/definitions/mk20.UploadStatusCode' summary: Status of deal upload post: + consumes: + - application/json description: Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal. parameters: @@ -752,6 +858,12 @@ paths: name: id required: true type: string + - description: Metadata for initiating an upload operation + in: body + name: data + required: true + schema: + $ref: '#/definitions/mk20.StartUpload' responses: "200": description: UploadStartCodeOk indicates a successful upload start request @@ -795,7 +907,7 @@ paths: type: string - description: raw binary in: body - name: body + name: data required: true schema: items: @@ -922,11 +1034,10 @@ securityDefinitions: - `PublicKeyType`: String representation of type of wallet (e.g., "ed25519", "bls", "secp256k1") - `PublicKey`: Base64 string of public key bytes - `Signature`: Signature is Base64 string of signature bytes. - - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following three components, in order. + - The client is expected to sign the SHA-256 hash of a message constructed by concatenating the following components, in order. - The raw public key bytes (not a human-readable address) - - The HTTP request path, such as /user/info - - The timestamp, truncated to the nearest minute, formatted in RFC3339 (e.g., 2025-07-15T17:42:00Z) - - These three byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. + - The timestamp, truncated to the nearest hour, formatted in RFC3339 (e.g., 2025-07-15T17:00:00Z) + - These two byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. in: header name: Authorization type: apiKey diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 4dbe232b9..625f273d5 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "net/http" "runtime" "runtime/debug" "sync/atomic" @@ -143,7 +142,14 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti } } - return m.processPDPDeal(ctx, deal) + if deal.Products.PDPV1 != nil { + return m.processPDPDeal(ctx, deal) + } + + return &ProviderDealRejectionInfo{ + HTTPCode: ErrUnsupportedProduct, + Reason: "Unsupported product", + } } func (m *MK20) processDDODeal(ctx context.Context, deal *Deal, tx *harmonydb.Tx) *ProviderDealRejectionInfo { @@ -257,13 +263,27 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - if deal.Data != nil { + if deal.Data == nil { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, Reason: "Data Source must be defined for a DDO deal", }, nil } + if deal.Products.RetrievalV1 == nil { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Retrieval product must be defined for a DDO deal", + }, nil + } + + if deal.Products.RetrievalV1.AnnouncePiece { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Piece cannot be announced for a DDO deal", + }, nil + } + size, err := deal.Size() if err != nil { log.Errorw("error getting deal size", "deal", deal, "error", err) @@ -368,6 +388,15 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe } func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + log.Errorf("panic occurred in PDP: %v\n%s", r, trace[:n]) + debug.PrintStack() + } + }() + rejection, err := m.sanitizePDPDeal(ctx, deal) if err != nil { log.Errorw("PDP deal rejected", "deal", deal, "error", err) @@ -434,8 +463,8 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } if pdp.DeleteRoot { - n, err := m.DB.Exec(ctx, `INSERT INTO pdp_delete_root (id, client, set_id, roots, extra_data) VALUES ($1, $2, $3, $4, $5)`, - deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.ExtraData) + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_root_delete (id, client, set_id, roots, extra_data) VALUES ($1, $2, $3, $4, $5)`, + deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.RootIDs, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP delete root: %w", err) } @@ -459,23 +488,93 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } } log.Debugw("PDP deal inserted in DB", "deal", deal.Identifier.String()) - return nil + return &ProviderDealRejectionInfo{ + HTTPCode: Ok, + } } func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { + if deal.Products.PDPV1.AddRoot && deal.Products.RetrievalV1 == nil { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Retrieval deal is required for pdp_v1", + }, nil + } + if deal.Data != nil { if deal.Data.SourceOffline != nil { return &ProviderDealRejectionInfo{ - HTTPCode: http.StatusBadRequest, + HTTPCode: ErrBadProposal, Reason: "Offline data source is not supported for pdp_v1", }, nil } + + if deal.Data.Format.Raw != nil && deal.Products.RetrievalV1.AnnouncePayload { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Raw bytes deal cannot be announced to IPNI", + }, nil + } } + + p := deal.Products.PDPV1 + + // This serves as Auth for now. We are checking if client is authorized to make changes to the proof set or roots + // In future this will be replaced by an ACL check + + if p.DeleteProofSet || p.AddRoot { + pid := *p.ProofSetID + var exists bool + err := m.DB.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE AND client = $2)`, pid, deal.Client.String()).Scan(&exists) + if err != nil { + log.Errorw("error checking if proofset exists", "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", + }, nil + } + if !exists { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "proofset does not exist for the client", + }, nil + } + } + + if p.DeleteRoot { + pid := *p.ProofSetID + var exists bool + err := m.DB.QueryRow(ctx, `SELECT COUNT(*) = cardinality($2::BIGINT[]) AS all_exist_and_active + FROM pdp_proofset_root r + JOIN pdp_proof_set s ON r.proof_set_id = s.id + WHERE r.proof_set_id = $1 + AND r.root = ANY($2) + AND r.removed = FALSE + AND s.removed = FALSE + AND r.client = $3 + AND s.client = $3;`, pid, p.RootIDs, deal.Client.String()).Scan(&exists) + if err != nil { + log.Errorw("error checking if proofset and roots exist for the client", "error", err) + return &ProviderDealRejectionInfo{ + HTTPCode: ErrServerInternalError, + Reason: "", + }, nil + + } + if !exists { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "proofset or one of the roots does not exist for the client", + }, nil + } + } + return nil, nil } func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error { pdp := deal.Products.PDPV1 + retv := deal.Products.RetrievalV1 data := deal.Data dealID := deal.Identifier.String() pi, err := deal.PieceInfo() @@ -541,10 +640,10 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error n, err = tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, deal_aggregation) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + extra_data, deal_aggregation, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, dealID, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, aggregation) + pdp.ExtraData, aggregation, retv.Indexing, retv.AnnouncePayload) if err != nil { return xerrors.Errorf("inserting PDP pipeline: %w", err) } @@ -644,10 +743,10 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error } pBatch.Queue(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, - proof_set_id, extra_data, piece_ref, deal_aggregation, aggr_index) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + proof_set_id, extra_data, piece_ref, deal_aggregation, aggr_index, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, dealID, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), spi.Size, spi.RawSize, - pdp.ExtraData, *pdp.ProofSetID, aggregation, i) + pdp.ExtraData, *pdp.ProofSetID, aggregation, i, retv.Indexing, retv.AnnouncePayload) if pBatch.Len() > pBatchSize { res := tx.SendBatch(ctx, pBatch) if err := res.Close(); err != nil { diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index cab3619c2..7e5e739aa 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -917,14 +917,15 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } pdp := uDeal.Products.PDPV1 + retv := uDeal.Products.RetrievalV1 // Insert the PDP pipeline n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type) + pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index bd968a80f..6ee90a978 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -26,16 +26,16 @@ type PDPV1 struct { DeleteRoot bool `json:"delete_root"` // ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. - ProofSetID *uint64 `json:"proof_set_id"` + ProofSetID *uint64 `json:"proof_set_id,omitempty"` // RecordKeeper specifies the record keeper contract address for the new PDP proofset. RecordKeeper string `json:"record_keeper"` // RootIDs is a list of root ids in a proof set. - RootIDs []uint64 `json:"root_ids"` + RootIDs []uint64 `json:"root_ids,omitempty"` // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. - ExtraData []byte `json:"extra_data"` + ExtraData []byte `json:"extra_data,omitempty"` } func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { @@ -55,44 +55,11 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er if p.RecordKeeper == "" { return ErrBadProposal, xerrors.Errorf("record_keeper must be defined for create_proof_set") } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for create_proof_set") - } if !common.IsHexAddress(p.RecordKeeper) { return ErrBadProposal, xerrors.Errorf("record_keeper must be a valid address") } } - if p.DeleteProofSet { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_proof_set") - } - } - - if p.AddRoot { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for add_root") - } - } - - if p.DeleteRoot { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") - } - if len(p.RootIDs) == 0 { - return ErrBadProposal, xerrors.Errorf("root_ids must be defined for delete_proof_set") - } - } - // Only 1 action is allowed per deal if btoi(p.CreateProofSet)+btoi(p.DeleteProofSet)+btoi(p.AddRoot)+btoi(p.DeleteRoot) > 1 { return ErrBadProposal, xerrors.Errorf("only one action is allowed per deal") @@ -101,6 +68,9 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er ctx := context.Background() if p.DeleteProofSet { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") + } pid := *p.ProofSetID var exists bool err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) @@ -108,14 +78,14 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset does not exist") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_proof_set") + return ErrBadProposal, xerrors.Errorf("proofset does not exist for the client") } } if p.AddRoot { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") + } pid := *p.ProofSetID var exists bool err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) @@ -123,32 +93,31 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset does not exist") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for add_root") + return ErrBadProposal, xerrors.Errorf("proofset does not exist for the client") } } if p.DeleteRoot { + if p.ProofSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") + } pid := *p.ProofSetID + if len(p.RootIDs) == 0 { + return ErrBadProposal, xerrors.Errorf("root_ids must be defined for delete_proof_set") + } var exists bool err := db.QueryRow(ctx, `SELECT COUNT(*) = cardinality($2::BIGINT[]) AS all_exist_and_active FROM pdp_proofset_root r - JOIN pdp_proof_set s ON r.proofset = s.id - WHERE r.proofset = $1 + JOIN pdp_proof_set s ON r.proof_set_id = s.id + WHERE r.proof_set_id = $1 AND r.root = ANY($2) AND r.removed = FALSE - AND s.removed = FALSE; - )`, pid, p.RootIDs).Scan(&exists) + AND s.removed = FALSE;`, pid, p.RootIDs).Scan(&exists) if err != nil { return ErrServerInternalError, xerrors.Errorf("checking if proofset and roots exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset or one of the roots does not exist") - } - if len(p.ExtraData) == 0 { - return ErrBadProposal, xerrors.Errorf("extra_data must be defined for delete_root") + return ErrBadProposal, xerrors.Errorf("proofset or one of the roots does not exist for the client") } } diff --git a/market/mk20/retrieval_v1.go b/market/mk20/retrieval_v1.go index 500263780..bbd316962 100644 --- a/market/mk20/retrieval_v1.go +++ b/market/mk20/retrieval_v1.go @@ -28,6 +28,10 @@ func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCo if !r.Indexing && r.AnnouncePayload { return ErrProductValidationFailed, xerrors.Errorf("deal cannot be announced to IPNI without indexing") } + + if r.AnnouncePiece && r.AnnouncePayload { + return ErrProductValidationFailed, xerrors.Errorf("cannot announce both payload and piece to IPNI at the same time") + } return Ok, nil } diff --git a/market/mk20/types.go b/market/mk20/types.go index 26c961bd2..48304f8cf 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -22,7 +22,7 @@ type Deal struct { Client address.Address `json:"client"` // Data represents the source of piece data and associated metadata. - Data *DataSource `json:"data"` + Data *DataSource `json:"data,omitempty"` // Products represents a collection of product-specific information associated with a deal Products Products `json:"products"` @@ -30,13 +30,13 @@ type Deal struct { type Products struct { // DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) - DDOV1 *DDOV1 `json:"ddo_v1"` + DDOV1 *DDOV1 `json:"ddo_v1,omitempty"` // RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags. - RetrievalV1 *RetrievalV1 `json:"retrieval_v1"` + RetrievalV1 *RetrievalV1 `json:"retrieval_v1,omitempty"` // PDPV1 represents product-specific configuration for PDP version 1 deals. - PDPV1 *PDPV1 `json:"pdp_v1"` + PDPV1 *PDPV1 `json:"pdp_v1,omitempty"` } // DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. @@ -49,16 +49,16 @@ type DataSource struct { Format PieceDataFormat `json:"format"` // SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. - SourceHTTP *DataSourceHTTP `json:"source_http"` + SourceHTTP *DataSourceHTTP `json:"source_http,omitempty"` // SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. - SourceAggregate *DataSourceAggregate `json:"source_aggregate"` + SourceAggregate *DataSourceAggregate `json:"source_aggregate,omitempty"` // SourceOffline defines the data source for offline pieces, including raw size information. - SourceOffline *DataSourceOffline `json:"source_offline"` + SourceOffline *DataSourceOffline `json:"source_offline,omitempty"` // SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import - SourceHttpPut *DataSourceHttpPut `json:"source_httpput"` + SourceHttpPut *DataSourceHttpPut `json:"source_httpput,omitempty"` // SourceStorageProvider -> sp IDs/ipni, pieceCids } @@ -67,13 +67,13 @@ type DataSource struct { type PieceDataFormat struct { // Car represents the optional CAR file format, including its metadata and versioning details. - Car *FormatCar `json:"car"` + Car *FormatCar `json:"car,omitempty"` // Aggregate holds a reference to the aggregated format of piece data. - Aggregate *FormatAggregate `json:"aggregate"` + Aggregate *FormatAggregate `json:"aggregate,omitempty"` // Raw represents the raw format of the piece data, encapsulated as bytes. - Raw *FormatBytes `json:"raw"` + Raw *FormatBytes `json:"raw,omitempty"` } // FormatCar represents the CAR (Content Addressable archive) format for piece data serialization. diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 1f11645f5..21dac6bf9 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -13,6 +13,8 @@ import ( "fmt" "net/http" "net/url" + "runtime" + "runtime/debug" "strings" "time" @@ -35,6 +37,15 @@ import ( ) func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + log.Errorf("panic occurred in validation: %v\n%s", r, trace[:n]) + debug.PrintStack() + } + }() + if d.Client.Empty() { return ErrBadProposal, xerrors.Errorf("no client") } @@ -335,12 +346,13 @@ func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, if err != nil { return code, err } - if d.RetrievalV1 == nil { - return ErrProductValidationFailed, xerrors.Errorf("retrieval v1 is required for pdp v1") - } - if d.RetrievalV1.Indexing || d.RetrievalV1.AnnouncePayload { - return ErrProductValidationFailed, xerrors.Errorf("payload indexing and announcement is not supported for pdp v1") - } + // TODO: Enable this once Indexing is done + //if d.RetrievalV1 == nil { + // return ErrProductValidationFailed, xerrors.Errorf("retrieval v1 is required for pdp v1") + //} + //if d.RetrievalV1.Indexing || d.RetrievalV1.AnnouncePayload { + // return ErrProductValidationFailed, xerrors.Errorf("payload indexing and announcement is not supported for pdp v1") + //} } if nproducts == 0 { @@ -355,16 +367,16 @@ func (d Products) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, } type DBDDOV1 struct { - DDO *DDOV1 `json:"ddo"` - DealID string `json:"deal_id"` - Complete bool `json:"complete"` - Error sql.NullString `json:"error"` + DDO *DDOV1 `json:"ddo"` + DealID int64 `json:"deal_id"` + Complete bool `json:"complete"` + Error string `json:"error"` } type DBPDPV1 struct { - PDP *PDPV1 `json:"pdp"` - Complete bool `json:"complete"` - Error sql.NullString `json:"error"` + PDP *PDPV1 `json:"pdp"` + Complete bool `json:"complete"` + Error string `json:"error"` } type DBDeal struct { @@ -637,6 +649,7 @@ type DealStatusResponse struct { ErrorMsg string `json:"error_msg"` } +// DealProductStatusResponse represents the status response for deal products with their respective deal statuses. type DealProductStatusResponse struct { // DDOV1 holds the DealStatusResponse for product "ddo_v1". @@ -740,13 +753,13 @@ func IsProductEnabled(db *harmonydb.DB, name ProductName) (DealCode, error) { return Ok, nil } -// SupportedProducts represents a collection of products supported by the SP. +// SupportedProducts represents array of products supported by the SP. type SupportedProducts struct { // Contracts represents a list of supported contract addresses in string format. Products []string `json:"products"` } -// SupportedDataSources represents a collection of dats sources supported by the SP. +// SupportedDataSources represents array of dats sources supported by the SP. type SupportedDataSources struct { // Contracts represents a list of supported contract addresses in string format. Sources []string `json:"sources"` @@ -846,12 +859,12 @@ func clientAllowed(ctx context.Context, db *harmonydb.DB, client string) (bool, const Authprefix = "CurioAuth " // Auth verifies the custom authentication header by parsing its contents and validating the signature using the provided database connection. -func Auth(header, path string, db *harmonydb.DB) (bool, string, error) { +func Auth(header string, db *harmonydb.DB) (bool, string, error) { keyType, pubKey, sig, err := parseCustomAuth(header) if err != nil { return false, "", xerrors.Errorf("parsing auth header: %w", err) } - return verifySignature(db, keyType, path, pubKey, sig) + return verifySignature(db, keyType, pubKey, sig) } func parseCustomAuth(header string) (keyType string, pubKey, sig []byte, err error) { @@ -887,15 +900,15 @@ func parseCustomAuth(header string) (keyType string, pubKey, sig []byte, err err return keyType, pubKey, sig, nil } -func verifySignature(db *harmonydb.DB, keyType string, path string, pubKey, signature []byte) (bool, string, error) { - now := time.Now().Truncate(time.Minute) - minus1 := now.Add(-1 * time.Minute) - plus1 := now.Add(1 * time.Minute) +func verifySignature(db *harmonydb.DB, keyType string, pubKey, signature []byte) (bool, string, error) { + now := time.Now().Truncate(time.Hour) + minus1 := now.Add(-59 * time.Minute) + plus1 := now.Add(59 * time.Minute) timeStamps := []time.Time{now, minus1, plus1} var msgs [][32]byte for _, t := range timeStamps { - msgs = append(msgs, sha256.Sum256(bytes.Join([][]byte{pubKey, []byte(path), []byte(t.Format(time.RFC3339))}, []byte{}))) + msgs = append(msgs, sha256.Sum256(bytes.Join([][]byte{pubKey, []byte(t.Format(time.RFC3339))}, []byte{}))) } switch keyType { diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index 10292f5be..b62c716f8 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -11,20 +11,31 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" +const PDPCalibnet = "0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC" +const PDPTestNet = "Change Me" + type PDPContracts struct { PDPVerifier common.Address } func ContractAddresses() PDPContracts { + return PDPContracts{ + PDPVerifier: ConfigurePDPAddress(), + } +} + +func ConfigurePDPAddress() common.Address { switch build.BuildType { case build.BuildCalibnet: - return PDPContracts{ - PDPVerifier: common.HexToAddress("0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC"), - } + return common.HexToAddress(PDPCalibnet) case build.BuildMainnet: - return PDPContracts{ - PDPVerifier: common.HexToAddress("0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6"), + return common.HexToAddress(PDPMainnet) + case build.Build2k, build.BuildDebug: + if !common.IsHexAddress(PDPTestNet) { + panic("PDPTestNet not set") } + return common.HexToAddress(PDPTestNet) default: panic("pdp contracts unknown for this network") } diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index 7a36a81a7..0959b185a 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -41,6 +41,9 @@ func (s *PipelineGC) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done if err := s.cleanupMK20DealPipeline(); err != nil { return false, xerrors.Errorf("cleanupMK20DealPipeline: %w", err) } + if err := s.cleanupPDPPipeline(); err != nil { + return false, xerrors.Errorf("cleanupPDPPipeline: %w", err) + } return true, nil } @@ -218,5 +221,14 @@ func (s *PipelineGC) cleanupUnseal() error { return nil } +func (s *PipelineGC) cleanupPDPPipeline() error { + ctx := context.Background() + _, err := s.db.Exec(ctx, `DELETE FROM pdp_pipeline WHERE complete = TRUE;`) + if err != nil { + return xerrors.Errorf("failed to clean up sealed deals: %w", err) + } + return nil +} + var _ harmonytask.TaskInterface = &PipelineGC{} var _ = harmonytask.Reg(&PipelineGC{}) diff --git a/tasks/gc/storage_gc_mark.go b/tasks/gc/storage_gc_mark.go index cd7516b71..25777b819 100644 --- a/tasks/gc/storage_gc_mark.go +++ b/tasks/gc/storage_gc_mark.go @@ -324,6 +324,9 @@ func (s *StorageGCMark) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d lb := policy.GetWinningPoStSectorSetLookback(nv) + builtin.EpochsInDay + 1 finalityHeight := head.Height() - lb + if finalityHeight < 0 { + finalityHeight = 1 + } finalityTipset, err := s.api.ChainGetTipSetByHeight(ctx, finalityHeight, head.Key()) if err != nil { diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index c32aa3299..e43205111 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -25,7 +25,7 @@ import ( "github.com/filecoin-project/curio/market/mk20" ) -const CheckIndexInterval = 9 * time.Minute +const CheckIndexInterval = time.Hour * 6 var MaxOngoingIndexingTasks = 40 @@ -91,7 +91,7 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask SELECT mm.piece_cid, mpd.piece_length, mpd.piece_offset, mpd.sp_id, mpd.sector_num, mpd.raw_size, mpd.piece_ref, mpd.id FROM market_piece_metadata mm LEFT JOIN market_piece_deal mpd ON mm.piece_cid = mpd.piece_cid AND mm.piece_size = mpd.piece_length - WHERE mm.indexed = true + WHERE mm.indexed = true AND mpd.sp_id > 0 AND mpd.sector_num > 0 `) if err != nil { return err diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index a30441595..f5ed21882 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -695,8 +695,6 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T } } - log.Infow("mk20 tasks", "tasks", mk20tasks) - if storiface.FTUnsealed != 1 { panic("storiface.FTUnsealed != 1") } @@ -711,12 +709,8 @@ func (i *IndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.T return nil, xerrors.Errorf("getting mk12 tasks: %w", err) } - log.Infow("mk12 tasks", "tasks", mk12tasks) - tasks = append(mk20tasks, mk12tasks...) - log.Infow("tasks", "tasks", tasks) - ls, err := i.sc.LocalStorage(ctx) if err != nil { return nil, xerrors.Errorf("getting local storage: %w", err) diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index ebf9081a7..0f7c07992 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -49,7 +49,6 @@ var ilog = logging.Logger("ipni") type IPNITask struct { db *harmonydb.DB - indexStore *indexstore.IndexStore pieceProvider *pieceprovider.SectorReader cpr *cachedreader.CachedPieceReader sc *ffi.SealCalls @@ -57,10 +56,9 @@ type IPNITask struct { max taskhelp.Limiter } -func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, pieceProvider *pieceprovider.SectorReader, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { +func NewIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, pieceProvider *pieceprovider.SectorReader, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *IPNITask { return &IPNITask{ db: db, - indexStore: indexStore, pieceProvider: pieceProvider, cpr: cpr, sc: sc, diff --git a/tasks/indexing/task_pdp_indexing.go b/tasks/indexing/task_pdp_indexing.go new file mode 100644 index 000000000..d44efe159 --- /dev/null +++ b/tasks/indexing/task_pdp_indexing.go @@ -0,0 +1,361 @@ +package indexing + +import ( + "context" + "errors" + "time" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/mk20" +) + +type PDPIndexingTask struct { + db *harmonydb.DB + indexStore *indexstore.IndexStore + cpr *cachedreader.CachedPieceReader + sc *ffi.SealCalls + cfg *config.CurioConfig + insertConcurrency int + insertBatchSize int + max taskhelp.Limiter +} + +func NewPDPIndexingTask(db *harmonydb.DB, sc *ffi.SealCalls, indexStore *indexstore.IndexStore, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *PDPIndexingTask { + + return &PDPIndexingTask{ + db: db, + indexStore: indexStore, + cpr: cpr, + sc: sc, + cfg: cfg, + insertConcurrency: cfg.Market.StorageMarketConfig.Indexing.InsertConcurrency, + insertBatchSize: cfg.Market.StorageMarketConfig.Indexing.InsertBatchSize, + max: max, + } +} + +func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var tasks []struct { + ID string `db:"id"` + PieceCIDV2 string `db:"piece_cid_v2"` + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + PieceRef int64 `db:"piece_ref"` + Indexing bool `db:"indexing"` + } + + err = P.db.Select(ctx, &tasks, `SELECT id, piece_cid_v2, piece_cid, piece_size, raw_size, piece_ref, indexing FROM pdp_pipeline WHERE indexing_task_id = $1 AND indexed = FALSE`, taskID) + if err != nil { + return false, xerrors.Errorf("getting PDP pending indexing tasks: %w", err) + } + + if len(tasks) != 1 { + return false, xerrors.Errorf("incorrect rows for pending indexing tasks: %d", len(tasks)) + } + + task := tasks[0] + + var indexed bool + err = P.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1 and piece_size = $2`, task.PieceCID, task.PieceSize).Scan(&indexed) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("checking if piece %s is already indexed: %w", task.PieceCIDV2, err) + } + + pcid2, err := cid.Parse(task.PieceCIDV2) + if err != nil { + return false, xerrors.Errorf("parsing piece CID: %w", err) + } + + id, err := ulid.Parse(task.ID) + if err != nil { + return false, xerrors.Errorf("parsing task id: %w", err) + } + + deal, err := mk20.DealFromDB(ctx, P.db, id) + if err != nil { + return false, xerrors.Errorf("getting deal from db: %w", err) + } + + var subPieces []mk20.DataSource + var byteData bool + + if deal.Data.Format.Aggregate != nil { + if deal.Data.Format.Aggregate.Type > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + } + } + + if deal.Data.Format.Raw != nil { + byteData = true + } + + if indexed || !task.Indexing || byteData { + err = P.recordCompletion(ctx, taskID, task.ID, task.PieceCID, task.PieceSize, task.RawSize, task.PieceRef, false) + if err != nil { + return false, err + } + log.Infow("Piece already indexed or should not be indexed", "piece_cid", task.PieceCIDV2, "indexed", indexed, "should_index", task.Indexing, "id", task.ID, "sp_id") + + return true, nil + } + + reader, _, err := P.cpr.GetSharedPieceReader(ctx, pcid2) + + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + + defer reader.Close() + + startTime := time.Now() + + dealCfg := P.cfg.Market.StorageMarketConfig + chanSize := dealCfg.Indexing.InsertConcurrency * dealCfg.Indexing.InsertBatchSize + + recs := make(chan indexstore.Record, chanSize) + var blocks int64 + + var eg errgroup.Group + addFail := make(chan struct{}) + var interrupted bool + + eg.Go(func() error { + defer close(addFail) + return P.indexStore.AddIndex(ctx, pcid2, recs) + }) + + var aggidx map[cid.Cid][]indexstore.Record + + if len(subPieces) > 0 { + blocks, aggidx, interrupted, err = IndexAggregate(pcid2, reader, abi.PaddedPieceSize(task.PieceSize), subPieces, recs, addFail) + } else { + blocks, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) + } + + if err != nil { + // Indexing itself failed, stop early + close(recs) // still safe to close, AddIndex will exit on channel close + // wait for AddIndex goroutine to finish cleanly + _ = eg.Wait() + return false, xerrors.Errorf("indexing failed: %w", err) + } + + // Close the channel + close(recs) + + // Wait till AddIndex is finished + err = eg.Wait() + if err != nil { + return false, xerrors.Errorf("adding index to DB (interrupted %t): %w", interrupted, err) + } + + log.Infof("Indexing deal %s took %0.3f seconds", task.ID, time.Since(startTime).Seconds()) + + // Save aggregate index if present + for k, v := range aggidx { + if len(v) > 0 { + err = P.indexStore.InsertAggregateIndex(ctx, k, v) + if err != nil { + return false, xerrors.Errorf("inserting aggregate index: %w", err) + } + } + } + + err = P.recordCompletion(ctx, taskID, task.ID, task.PieceCID, task.PieceSize, task.RawSize, task.PieceRef, true) + if err != nil { + return false, err + } + + blocksPerSecond := float64(blocks) / time.Since(startTime).Seconds() + log.Infow("Piece indexed", "piece_cid", task.PieceCIDV2, "id", task.ID, "blocks", blocks, "blocks_per_second", blocksPerSecond) + + return true, nil +} + +func (P *PDPIndexingTask) recordCompletion(ctx context.Context, taskID harmonytask.TaskID, id, PieceCID string, size, rawSize, pieceRef int64, indexed bool) error { + comm, err := P.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + _, err = tx.Exec(`SELECT process_piece_deal($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + id, PieceCID, false, -1, -1, nil, size, rawSize, indexed, pieceRef, false, 0) + if err != nil { + return false, xerrors.Errorf("failed to update piece metadata and piece deal for deal %s: %w", id, err) + } + + if P.cfg.Market.StorageMarketConfig.IPNI.Disable { + n, err := P.db.Exec(ctx, `UPDATE pdp_pipeline SET indexed = TRUE, indexing_task_id = NULL, + complete = TRUE WHERE id = $1 AND indexing_task_id = $2`, id, taskID) + if err != nil { + return false, xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store indexing success: updated %d rows", n) + } + } else { + n, err := tx.Exec(`UPDATE pdp_pipeline SET indexed = TRUE, indexing_task_id = NULL + WHERE id = $1 AND indexing_task_id = $2`, id, taskID) + if err != nil { + return false, xerrors.Errorf("store indexing success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store indexing success: updated %d rows", n) + } + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("committing transaction: %w", err) + } + if !comm { + return xerrors.Errorf("failed to commit transaction") + } + + return nil +} + +func (P *PDPIndexingTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + ctx := context.Background() + + type task struct { + TaskID harmonytask.TaskID `db:"indexing_task_id"` + StorageID string `db:"storage_id"` + PieceRef int64 `db:"piece_ref"` + Indexing bool `db:"indexing"` + } + + indIDs := make([]int64, len(ids)) + for x, id := range ids { + indIDs[x] = int64(id) + } + + var tasks []*task + if storiface.FTPiece != 32 { + panic("storiface.FTPiece != 32") + } + + err := P.db.Select(ctx, &tasks, `SELECT indexing_task_id, piece_ref, indexing FROM pdp_pipeline WHERE indexing_task_id = ANY($1)`, indIDs) + if err != nil { + return nil, xerrors.Errorf("getting PDP indexing details: %w", err) + } + + for _, t := range tasks { + + if !t.Indexing { + continue + } + + var sLocation string + err = P.db.QueryRow(ctx, ` + SELECT sl.storage_id + FROM parked_piece_refs ppr + JOIN sector_location sl + ON sl.sector_num = ppr.piece_id + AND sl.miner_id = 0 + AND sl.sector_filetype = 32 + WHERE ppr.ref_id = $1 + `, t.PieceRef).Scan(&sLocation) + if err != nil { + return nil, xerrors.Errorf("getting storage_id: %w", err) + } + + t.StorageID = sLocation + } + + ls, err := P.sc.LocalStorage(ctx) + if err != nil { + return nil, xerrors.Errorf("getting local storage: %w", err) + } + + localStorageMap := make(map[string]bool, len(ls)) + for _, l := range ls { + localStorageMap[string(l.ID)] = true + } + + for _, t := range tasks { + if !t.Indexing { + return &t.TaskID, nil + } + if found, ok := localStorageMap[t.StorageID]; ok && found { + return &t.TaskID, nil + } + } + + return nil, nil +} + +func (P *PDPIndexingTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "PDPIndexing", + Cost: resources.Resources{ + Cpu: 1, + Ram: uint64(P.insertBatchSize * P.insertConcurrency * 56 * 2), + }, + Max: P.max, + MaxFailures: 3, + IAmBored: passcall.Every(3*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return P.schedule(context.Background(), taskFunc) + }), + } +} + +func (P *PDPIndexingTask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + // schedule submits + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var pendings []struct { + ID string `db:"id"` + } + + err := tx.Select(&pendings, `SELECT id FROM pdp_pipeline + WHERE after_save_cache = TRUE + AND indexing_task_id IS NULL + AND indexed = FALSE + ORDER BY indexing_created_at ASC LIMIT 1;`) + if err != nil { + return false, xerrors.Errorf("getting PDP pending indexing tasks: %w", err) + } + + if len(pendings) == 0 { + return false, nil + } + + pending := pendings[0] + _, err = tx.Exec(`UPDATE pdp_pipeline SET indexing_task_id = $1 + WHERE indexing_task_id IS NULL AND id = $2`, id, pending.ID) + if err != nil { + return false, xerrors.Errorf("updating PDP indexing task id: %w", err) + } + + stop = false + return true, nil + }) + } + + return nil +} + +func (P *PDPIndexingTask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &PDPIndexingTask{} +var _ = harmonytask.Reg(&PDPIndexingTask{}) diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go new file mode 100644 index 000000000..9f8709a5e --- /dev/null +++ b/tasks/indexing/task_pdp_ipni.go @@ -0,0 +1,508 @@ +package indexing + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/ipfs/go-cid" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipni/go-libipni/ingest/schema" + "github.com/ipni/go-libipni/maurl" + "github.com/ipni/go-libipni/metadata" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/oklog/ulid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/build" + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/cachedreader" + "github.com/filecoin-project/curio/lib/commcidv2" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/ipni/chunker" + "github.com/filecoin-project/curio/market/ipni/ipniculib" + "github.com/filecoin-project/curio/market/mk20" +) + +type PDPIPNITask struct { + db *harmonydb.DB + cpr *cachedreader.CachedPieceReader + sc *ffi.SealCalls + cfg *config.CurioConfig + max taskhelp.Limiter +} + +func NewPDPIPNITask(db *harmonydb.DB, sc *ffi.SealCalls, cpr *cachedreader.CachedPieceReader, cfg *config.CurioConfig, max taskhelp.Limiter) *PDPIPNITask { + return &PDPIPNITask{ + db: db, + cpr: cpr, + sc: sc, + cfg: cfg, + max: max, + } +} + +func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var tasks []struct { + ID string `db:"id"` + CtxID []byte `db:"context_id"` + Rm bool `db:"is_rm"` + Prov string `db:"provider"` + Complete bool `db:"complete"` + } + + err = P.db.Select(ctx, &tasks, `SELECT + id, + context_id, + is_rm, + provider, + complete + FROM + pdp_ipni_task + WHERE + task_id = $1;`, taskID) + if err != nil { + return false, xerrors.Errorf("getting ipni task params: %w", err) + } + + if len(tasks) != 1 { + return false, xerrors.Errorf("expected 1 ipni task params, got %d", len(tasks)) + } + + task := tasks[0] + + if task.Complete { + log.Infow("IPNI task already complete", "task_id", taskID) + return true, nil + } + + var pi abi.PieceInfo + err = pi.UnmarshalCBOR(bytes.NewReader(task.CtxID)) + if err != nil { + return false, xerrors.Errorf("unmarshaling piece info: %w", err) + } + + var rawSize abi.UnpaddedPieceSize + err = P.db.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1`, pi.PieceCID.String(), pi.Size).Scan(&rawSize) + if err != nil { + return false, xerrors.Errorf("querying raw size: %w", err) + } + + pcid2, err := commcidv2.PieceCidV2FromV1(pi.PieceCID, uint64(rawSize)) + if err != nil { + return false, xerrors.Errorf("getting piece CID v2: %w", err) + } + + reader, _, err := P.cpr.GetSharedPieceReader(ctx, pcid2) + if err != nil { + return false, xerrors.Errorf("getting piece reader from piece park: %w", err) + } + + defer reader.Close() + + recs := make(chan indexstore.Record, 1) + + var eg errgroup.Group + addFail := make(chan struct{}) + var interrupted bool + var subPieces []mk20.DataSource + chk := chunker.NewInitialChunker() + + eg.Go(func() error { + defer close(addFail) + for rec := range recs { + serr := chk.Accept(rec.Cid.Hash(), int64(rec.Offset), rec.Size) + if serr != nil { + addFail <- struct{}{} + return serr + } + } + return nil + }) + + id, serr := ulid.Parse(task.ID) + if serr != nil { + return false, xerrors.Errorf("parsing task id: %w", serr) + } + deal, serr := mk20.DealFromDB(ctx, P.db, id) + if serr != nil { + return false, xerrors.Errorf("getting deal from db: %w", serr) + } + + if deal.Data.Format.Raw != nil { + return false, xerrors.Errorf("raw data not supported") + } + + if deal.Data.Format.Car != nil { + _, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) + } + + if deal.Data.Format.Aggregate != nil { + if deal.Data.Format.Aggregate.Type > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.Size, subPieces, recs, addFail) + } + } + + if err != nil { + // Chunking itself failed, stop early + close(recs) // still safe to close, chk.Accept() will exit on channel close + // wait for chk.Accept() goroutine to finish cleanly + _ = eg.Wait() + return false, xerrors.Errorf("chunking failed: %w", err) + } + + // Close the channel + close(recs) + + // Wait till is finished + err = eg.Wait() + if err != nil { + return false, xerrors.Errorf("adding index to chunk (interrupted %t): %w", interrupted, err) + } + + // make sure we still own the task before writing to the database + if !stillOwned() { + return false, nil + } + + lnk, err := chk.Finish(ctx, P.db, pcid2) + if err != nil { + return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + } + + // make sure we still own the task before writing ad chains + if !stillOwned() { + return false, nil + } + + _, err = P.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var prev string + err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("querying previous head: %w", err) + } + + mds := metadata.IpfsGatewayHttp{} + md, err := mds.MarshalBinary() + if err != nil { + return false, xerrors.Errorf("marshaling metadata: %w", err) + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&privKey) + if err != nil { + return false, xerrors.Errorf("failed to get private ipni-libp2p key for PDP: %w", err) + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + adv := schema.Advertisement{ + Provider: task.Prov, + Entries: lnk, + ContextID: task.CtxID, + Metadata: md, + IsRm: task.Rm, + } + + { + u, err := url.Parse(fmt.Sprintf("https://%s", P.cfg.HTTP.DomainName)) + if err != nil { + return false, xerrors.Errorf("parsing announce address domain: %w", err) + } + if build.BuildType != build.BuildMainnet && build.BuildType != build.BuildCalibnet { + ls := strings.Split(P.cfg.HTTP.ListenAddress, ":") + u, err = url.Parse(fmt.Sprintf("http://%s:%s", P.cfg.HTTP.DomainName, ls[1])) + if err != nil { + return false, xerrors.Errorf("parsing announce address domain: %w", err) + } + } + + addr, err := maurl.FromURL(u) + if err != nil { + return false, xerrors.Errorf("converting URL to multiaddr: %w", err) + } + + adv.Addresses = append(adv.Addresses, addr.String()) + } + + if prev != "" { + prevCID, err := cid.Parse(prev) + if err != nil { + return false, xerrors.Errorf("parsing previous CID: %w", err) + } + + adv.PreviousID = cidlink.Link{Cid: prevCID} + } + + err = adv.Sign(pkey) + if err != nil { + return false, xerrors.Errorf("signing the advertisement: %w", err) + } + + err = adv.Validate() + if err != nil { + return false, xerrors.Errorf("validating the advertisement: %w", err) + } + + adNode, err := adv.ToNode() + if err != nil { + return false, xerrors.Errorf("converting advertisement to node: %w", err) + } + + ad, err := ipniculib.NodeToLink(adNode, schema.Linkproto) + if err != nil { + return false, xerrors.Errorf("converting advertisement to link: %w", err) + } + + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, pi.PieceCID.String(), pi.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), + adv.Signature, adv.Entries.String()) + + if err != nil { + return false, xerrors.Errorf("adding advertisement to the database: %w", err) + } + + n, err := tx.Exec(`UPDATE pdp_ipni_task SET complete = true WHERE task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to mark IPNI task complete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updated %d rows", n) + } + + return true, nil + + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("store IPNI success: %w", err) + } + + log.Infow("IPNI task complete", "task_id", taskID) + + return true, nil +} + +func (P *PDPIPNITask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (P *PDPIPNITask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "PDPIPNI", + Cost: resources.Resources{ + Cpu: 1, + Ram: 1 << 30, + }, + MaxFailures: 3, + IAmBored: passcall.Every(30*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return P.schedule(context.Background(), taskFunc) + }), + Max: P.max, + } +} + +func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + if P.cfg.Market.StorageMarketConfig.IPNI.Disable { + return nil + } + + // schedule submits + var stop bool + for !stop { + var markComplete *string + + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var pendings []struct { + ID string `db:"id"` + PieceCid string `db:"piece_cid"` + Size abi.UnpaddedPieceSize `db:"piece_size"` + RawSize abi.UnpaddedPieceSize `db:"raw_size"` + Index bool `db:"indexing"` + Announce bool `db:"announce"` + IndexingCreatedAt time.Time `db:"indexing_created_at"` + } + + err := tx.Select(&pendings, `SELECT + id, + piece_cid, + piece_size, + raw_size, + indexing, + announce, + indexing_created_at + FROM pdp_pipeline + WHERE indexed = TRUE + AND complete = FALSE + LIMIT 1;`) + if err != nil { + return false, xerrors.Errorf("getting pending IPNI announcing tasks: %w", err) + } + + if len(pendings) == 0 { + return false, nil + } + + p := pendings[0] + + // Skip IPNI if deal says not to announce or not to index (fast retrievals). If we announce without + // indexing, it will cause issue with retrievals. + if !p.Announce || !p.Index { + var n int + n, err = tx.Exec(`UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1`, p.ID) + + if err != nil { + return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("store IPNI success: updated %d rows", n) + } + + n, err = tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, p.ID) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + + stop = false // we found a task to schedule, keep going + return true, nil + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&privKey) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("failed to get private libp2p key for PDP: %w", err) + } + + // generate the ipni provider key + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + if err != nil { + return false, xerrors.Errorf("failed to generate a new key: %w", err) + } + + privKey, err = crypto.MarshalPrivateKey(pk) + if err != nil { + return false, xerrors.Errorf("failed to marshal the private key: %w", err) + } + + pid, err := peer.IDFromPublicKey(pk.GetPublic()) + if err != nil { + return false, xerrors.Errorf("getting peer ID: %w", err) + } + + n, err := tx.Exec(`INSERT INTO ipni_peerid (sp_id, priv_key, peer_id) VALUES ($1, $2, $3) ON CONFLICT(sp_id) DO NOTHING `, -1, privKey, pid.String()) + if err != nil { + return false, xerrors.Errorf("failed to to insert the key into DB: %w", err) + } + + if n == 0 { + return false, xerrors.Errorf("failed to insert the key into db") + } + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + pid, err := peer.IDFromPublicKey(pkey.GetPublic()) + if err != nil { + return false, fmt.Errorf("getting peer ID: %w", err) + } + + pcid, err := cid.Parse(p.PieceCid) + if err != nil { + return false, xerrors.Errorf("parsing piece CID: %w", err) + } + + pi := abi.PieceInfo{ + PieceCID: pcid, + Size: abi.PaddedPieceSize(p.Size), + } + + b := new(bytes.Buffer) + err = pi.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + ilog.Infof("Another IPNI announce task already present for piece %s in deal %s", p.PieceCid, p.ID) + // SET "complete" status to true for this deal, so it is not considered next time + markComplete = &p.ID + stop = false // we found a sector to work on, keep going + return true, nil + } + if strings.Contains(err.Error(), "already published") { + ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.ID) + // SET "complete" status to true for this deal, so it is not considered next time + markComplete = &p.ID + stop = false // we found a sector to work on, keep going + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + markComplete = &p.ID + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + if markComplete != nil { + n, err := P.db.Exec(ctx, `UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1 AND complete = FALSE`, *markComplete) + if err != nil { + log.Errorf("store IPNI success: updating pipeline: %s", err) + } + if n != 1 { + log.Errorf("store IPNI success: updated %d rows", n) + } + + n, err = P.db.Exec(ctx, `UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, *markComplete) + if err != nil { + log.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + log.Errorf("expected 1 row to be updated, got %d", n) + } + } + } + + return nil +} + +func (P *PDPIPNITask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ harmonytask.TaskInterface = &PDPIPNITask{} +var _ = harmonytask.Reg(&PDPIPNITask{}) diff --git a/tasks/pdp/proofset_addroot_watch.go b/tasks/pdp/proofset_addroot_watch.go index fcb32428d..d21e213a0 100644 --- a/tasks/pdp/proofset_addroot_watch.go +++ b/tasks/pdp/proofset_addroot_watch.go @@ -3,17 +3,16 @@ package pdp import ( "context" "encoding/json" + "errors" "fmt" "math/big" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ipfs/go-cid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/chainsched" - "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/pdp/contract" chainTypes "github.com/filecoin-project/lotus/chain/types" @@ -23,33 +22,22 @@ import ( type ProofSetRootAdd struct { ID string `db:"id"` Client string `db:"client"` - PieceCID string `db:"piece_cid"` // pieceCIDV2 - ProofSet uint64 `db:"proofset"` + PieceCID2 string `db:"piece_cid_v2"` // pieceCIDV2 + PieceCID string `db:"piece_cid"` + PieceSize int64 `db:"piece_size"` + RawSize int64 `db:"raw_size"` + ProofSet uint64 `db:"proof_set_id"` PieceRef int64 `db:"piece_ref"` AddMessageHash string `db:"add_message_hash"` AddMessageIndex int64 `db:"add_message_index"` } -// RootAddEntry represents entries from pdp_proofset_root_adds -type RootAddEntry struct { - ProofSet uint64 `db:"proofset"` - Root string `db:"root"` - AddMessageHash string `db:"add_message_hash"` - AddMessageIndex uint64 `db:"add_message_index"` - Subroot string `db:"subroot"` - SubrootOffset int64 `db:"subroot_offset"` - SubrootSize int64 `db:"subroot_size"` - PDPPieceRefID int64 `db:"pdp_pieceref"` - AddMessageOK *bool `db:"add_message_ok"` - PDPProofSetID uint64 `db:"proofset"` -} - // NewWatcherRootAdd sets up the watcher for proof set root additions -func NewWatcherRootAdd(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { +func NewWatcherRootAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetRootAdds(ctx, db, ethClient) + err := processPendingProofSetRootAdds(ctx, db) if err != nil { - log.Warnf("Failed to process pending proof set root adds: %v", err) + log.Errorf("Failed to process pending proof set root adds: %s", err) } return nil @@ -59,12 +47,12 @@ func NewWatcherRootAdd(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chain } // processPendingProofSetRootAdds processes root additions that have been confirmed on-chain -func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { +func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB) error { // Query for pdp_proofset_root_adds entries where add_message_ok = TRUE var rootAdds []ProofSetRootAdd err := db.Select(ctx, &rootAdds, ` - SELECT id, client, piece_cid, proofset, piece_ref, add_message_hash, add_message_index + SELECT id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, piece_ref, add_message_hash, add_message_index FROM pdp_pipeline WHERE after_add_root = TRUE AND after_add_root_msg = FALSE `) @@ -79,9 +67,9 @@ func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB, ethCl // Process each root addition for _, rootAdd := range rootAdds { - err := processProofSetRootAdd(ctx, db, ethClient, rootAdd) + err := processProofSetRootAdd(ctx, db, rootAdd) if err != nil { - log.Warnf("Failed to process root add for tx %s: %v", rootAdd.AddMessageHash, err) + log.Errorf("Failed to process root add for tx %s: %s", rootAdd.AddMessageHash, err) continue } } @@ -89,16 +77,17 @@ func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB, ethCl return nil } -func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client, rootAdd ProofSetRootAdd) error { +func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd ProofSetRootAdd) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool - err := db.QueryRow(ctx, ` - SELECT tx_success, tx_receipt - FROM message_waits_eth - WHERE signed_tx_hash = $1 - `, rootAdd.AddMessageHash).Scan(&txSuccess, &txReceiptJSON) + err := db.QueryRow(ctx, `SELECT tx_success, tx_receipt FROM message_waits_eth WHERE signed_tx_hash = $1 + AND tx_success IS NOT NULL + AND tx_receipt IS NOT NULL`, rootAdd.AddMessageHash).Scan(&txSuccess, &txReceiptJSON) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", rootAdd.AddMessageHash) + } return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", rootAdd.AddMessageHash, err) } @@ -140,15 +129,6 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *et return nil } - pcid, err := cid.Parse(rootAdd.PieceCID) - if err != nil { - return xerrors.Errorf("failed to parse piece CID: %w", err) - } - pi, err := mk20.GetPieceInfo(pcid) - if err != nil { - return xerrors.Errorf("failed to get piece info: %w", err) - } - // Get the ABI from the contract metadata pdpABI, err := contract.PDPVerifierMetaData.GetAbi() if err != nil { @@ -209,7 +189,7 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *et comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Update proof set for initialization upon first add _, err = tx.Exec(` - UPDATE pdp_proof_sets SET init_ready = true + UPDATE pdp_proof_set SET init_ready = true WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL `, rootAdd.ProofSet) if err != nil { @@ -219,7 +199,7 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *et // Insert into pdp_proofset_roots n, err := tx.Exec(` INSERT INTO pdp_proofset_root ( - proofset, + proof_set_id, client, piece_cid_v2, piece_cid, @@ -235,10 +215,10 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, ethClient *et `, rootAdd.ProofSet, rootAdd.Client, - pcid.String(), - pi.PieceCIDV1.String(), - pi.Size, - pi.RawSize, + rootAdd.PieceCID2, + rootAdd.PieceCID, + rootAdd.PieceSize, + rootAdd.RawSize, rootId, rootAdd.PieceRef, rootAdd.ID, diff --git a/tasks/pdp/proofset_create_watch.go b/tasks/pdp/proofset_create_watch.go index 908190b1c..1cdef1d92 100644 --- a/tasks/pdp/proofset_create_watch.go +++ b/tasks/pdp/proofset_create_watch.go @@ -3,12 +3,14 @@ package pdp import ( "context" "encoding/json" + "errors" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -28,7 +30,7 @@ func NewWatcherCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chains if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { err := processPendingProofSetCreates(ctx, db, ethClient) if err != nil { - log.Warnf("Failed to process pending proof set creates: %v", err) + log.Errorf("Failed to process pending proof set creates: %s", err) } return nil }); err != nil { @@ -41,7 +43,7 @@ func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethCli var proofSetCreates []ProofSetCreate err := db.Select(ctx, &proofSetCreates, ` - SELECT id, client, tx_hash, + SELECT id, client, tx_hash FROM pdp_proof_set_create WHERE tx_hash IS NOT NULL`) if err != nil { @@ -57,7 +59,7 @@ func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethCli for _, psc := range proofSetCreates { err := processProofSetCreate(ctx, db, psc, ethClient) if err != nil { - log.Warnf("Failed to process proof set create for tx %s: %v", psc.CreateMessageHash, err) + log.Errorf("Failed to process proof set create for tx %s: %s", psc.CreateMessageHash, err) continue } } @@ -69,12 +71,14 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool - err := db.QueryRow(ctx, ` - SELECT tx_success, tx_receipt - FROM message_waits_eth - WHERE signed_tx_hash = $1 - `, psc.CreateMessageHash).Scan(&txReceiptJSON, &txSuccess) + err := db.QueryRow(ctx, `SELECT tx_receipt, tx_success FROM message_waits_eth + WHERE signed_tx_hash = $1 + AND tx_success IS NOT NULL + AND tx_receipt IS NOT NULL`, psc.CreateMessageHash).Scan(&txReceiptJSON, &txSuccess) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", psc.CreateMessageHash) + } return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psc.CreateMessageHash, err) } diff --git a/tasks/pdp/proofset_delete_root_watch.go b/tasks/pdp/proofset_delete_root_watch.go index 3a26bbcb4..ea8453b9e 100644 --- a/tasks/pdp/proofset_delete_root_watch.go +++ b/tasks/pdp/proofset_delete_root_watch.go @@ -3,9 +3,10 @@ package pdp import ( "context" "encoding/json" + "errors" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -15,17 +16,17 @@ import ( ) type ProofSetRootDelete struct { - ID string `db:"id"` - ProofSet uint64 `db:"set_id"` - Roots int64 `db:"roots"` - Hash string `db:"tx_hash"` + ID string `db:"id"` + ProofSet uint64 `db:"set_id"` + Roots []int64 `db:"roots"` + Hash string `db:"tx_hash"` } -func NewWatcherRootDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { +func NewWatcherRootDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetRootDeletes(ctx, db, ethClient) + err := processPendingProofSetRootDeletes(ctx, db) if err != nil { - log.Warnf("Failed to process pending proof set creates: %v", err) + log.Errorf("Failed to process pending proof set creates: %s", err) } return nil }); err != nil { @@ -33,14 +34,12 @@ func NewWatcherRootDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *ch } } -func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { +func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB) error { var proofSetRootDeletes []ProofSetRootDelete err := db.Select(ctx, &proofSetRootDeletes, ` - SELECT id, tx_hash, roots, set_id - FROM pdp_delete_root - WHERE tx_hash IS NOT NULL`) + SELECT id, tx_hash, roots, set_id FROM pdp_root_delete WHERE tx_hash IS NOT NULL`) if err != nil { - return xerrors.Errorf("failed to select proof set deletes: %w", err) + return xerrors.Errorf("failed to select proof set root deletes: %w", err) } if len(proofSetRootDeletes) == 0 { @@ -48,9 +47,9 @@ func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB, et } for _, psd := range proofSetRootDeletes { - err := processProofSetRootDelete(ctx, db, psd, ethClient) + err := processProofSetRootDelete(ctx, db, psd) if err != nil { - log.Warnf("Failed to process proof set root delete for tx %s: %v", psd.Hash, err) + log.Errorf("Failed to process proof set root delete for tx %s: %s", psd.Hash, err) continue } } @@ -58,15 +57,16 @@ func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB, et return nil } -func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetRootDelete, ethClient *ethclient.Client) error { +func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetRootDelete) error { var txReceiptJSON []byte var txSuccess bool - err := db.QueryRow(ctx, ` - SELECT tx_success, tx_receipt - FROM message_waits_eth - WHERE signed_tx_hash = $1 - `, psd.Hash).Scan(&txReceiptJSON, &txSuccess) + err := db.QueryRow(ctx, `SELECT tx_receipt, tx_success FROM message_waits_eth WHERE signed_tx_hash = $1 + AND tx_success IS NOT NULL + AND tx_receipt IS NOT NULL`, psd.Hash).Scan(&txReceiptJSON, &txSuccess) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", psd.Hash) + } return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psd.Hash, err) } @@ -90,9 +90,9 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_delete_root WHERE id = $1`, psd.ID) + _, err = tx.Exec(`DELETE FROM pdp_root_delete WHERE id = $1`, psd.ID) if err != nil { - return false, xerrors.Errorf("failed to delete row from pdp_delete_root: %w", err) + return false, xerrors.Errorf("failed to delete row from pdp_root_delete: %w", err) } return true, nil }) @@ -109,7 +109,7 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS n, err := tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, remove_deal_id = $1, remove_message_hash = $2 - WHERE id = $3 AND root = ANY($4)`, psd.ID, psd.Hash, psd.ProofSet, psd.Roots) + WHERE proof_set_id = $3 AND root = ANY($4)`, psd.ID, psd.Hash, psd.ProofSet, psd.Roots) if err != nil { return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) } @@ -125,6 +125,10 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } + _, err = tx.Exec(`DELETE FROM pdp_root_delete WHERE id = $1`, psd.ID) + if err != nil { + return false, xerrors.Errorf("failed to delete row from pdp_root_delete: %w", err) + } return true, nil }) diff --git a/tasks/pdp/proofset_delete_watch.go b/tasks/pdp/proofset_delete_watch.go index 2dc070026..b3c094190 100644 --- a/tasks/pdp/proofset_delete_watch.go +++ b/tasks/pdp/proofset_delete_watch.go @@ -3,9 +3,10 @@ package pdp import ( "context" "encoding/json" + "errors" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -20,11 +21,11 @@ type ProofSetDelete struct { PID int64 `db:"set_id"` } -func NewWatcherDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { +func NewWatcherDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetDeletes(ctx, db, ethClient) + err := processPendingProofSetDeletes(ctx, db) if err != nil { - log.Warnf("Failed to process pending proof set creates: %v", err) + log.Errorf("Failed to process pending proof set creates: %s", err) } return nil }); err != nil { @@ -32,12 +33,12 @@ func NewWatcherDelete(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chains } } -func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { +func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB) error { // Query for pdp_proof_set_delete where txHash is not NULL var proofSetDeletes []ProofSetDelete err := db.Select(ctx, &proofSetDeletes, ` - SELECT id, client, tx_hash, + SELECT id, set_id, tx_hash FROM pdp_proof_set_delete WHERE tx_hash IS NOT NULL`) if err != nil { @@ -51,9 +52,9 @@ func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB, ethCli // Process each proof set delete for _, psd := range proofSetDeletes { - err := processProofSetDelete(ctx, db, psd, ethClient) + err := processProofSetDelete(ctx, db, psd) if err != nil { - log.Warnf("Failed to process proof set delete for tx %s: %v", psd.DeleteMessageHash, err) + log.Errorf("Failed to process proof set delete for tx %s: %s", psd.DeleteMessageHash, err) continue } } @@ -61,16 +62,17 @@ func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB, ethCli return nil } -func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDelete, ethClient *ethclient.Client) error { +func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDelete) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool - err := db.QueryRow(ctx, ` - SELECT tx_success, tx_receipt - FROM message_waits_eth - WHERE signed_tx_hash = $1 - `, psd.DeleteMessageHash).Scan(&txReceiptJSON, &txSuccess) + err := db.QueryRow(ctx, `SELECT tx_receipt, tx_success FROM message_waits_eth WHERE signed_tx_hash = $1 + AND tx_success IS NOT NULL + AND tx_receipt IS NOT NULL`, psd.DeleteMessageHash).Scan(&txReceiptJSON, &txSuccess) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", psd.DeleteMessageHash) + } return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psd.DeleteMessageHash, err) } @@ -114,6 +116,7 @@ func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDe } comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_proof_set SET removed = TRUE, remove_deal_id = $1, remove_message_hash = $2 @@ -124,10 +127,20 @@ func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDe if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } + + _, err = tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, + remove_deal_id = $1, + remove_message_hash = $2 + WHERE proof_set_id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) + } + _, err = tx.Exec(`DELETE FROM pdp_proof_set_delete WHERE id = $1`, psd.ID) if err != nil { return false, xerrors.Errorf("failed to delete row from pdp_proof_set_delete: %w", err) } + n, err = tx.Exec(`UPDATE market_mk20_deal SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) WHERE id = $1;`, psd.ID) @@ -137,16 +150,7 @@ func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDe if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - n, err = tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, - remove_deal_id = $1, - remove_message_hash = $2 - WHERE id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) - if err != nil { - return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) - } + return true, nil }) diff --git a/tasks/pdp/task_add_proofset.go b/tasks/pdp/task_add_proofset.go index ca0e9f3f5..2d228c41f 100644 --- a/tasks/pdp/task_add_proofset.go +++ b/tasks/pdp/task_add_proofset.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -49,7 +50,7 @@ func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() boo return false, xerrors.Errorf("failed to get task details from DB: %w", err) } - if len(pcreates) != 0 { + if len(pcreates) != 1 { return false, xerrors.Errorf("incorrect rows for proofset create found for taskID %d", taskID) } @@ -104,19 +105,34 @@ func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() boo // Insert into message_waits_eth and pdp_proofset_creates txHashLower := strings.ToLower(txHash.Hex()) - n, err := p.db.Exec(ctx, `UPDATE pdp_proof_set_create SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_proof_set_create SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_create: %d", n) + } + _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") + if err != nil { + return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to commit transaction: %w", err) } - if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_create: %d", n) + + if !comm { + return false, xerrors.Errorf("failed to commit transaction") } + return true, nil } func (p *PDPTaskAddProofSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { - //TODO implement me - panic("implement me") + return &ids[0], nil } func (p *PDPTaskAddProofSet) TypeDetails() harmonytask.TaskTypeDetails { @@ -141,8 +157,11 @@ func (p *PDPTaskAddProofSet) schedule(ctx context.Context, taskFunc harmonytask. stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_proof_set_create WHERE task_id IS NULL AND tx_hash IS NULL`).Scan(&id) + err := tx.QueryRow(`SELECT id FROM pdp_proof_set_create WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } return false, xerrors.Errorf("failed to query pdp_proof_set_create: %w", err) } if did == "" { @@ -181,3 +200,4 @@ func (p *PDPTaskAddProofSet) getSenderAddress(ctx context.Context) (common.Addre func (p *PDPTaskAddProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} var _ harmonytask.TaskInterface = &PDPTaskAddProofSet{} +var _ = harmonytask.Reg(&PDPTaskAddProofSet{}) diff --git a/tasks/pdp/task_addroot.go b/tasks/pdp/task_addroot.go index bae4a2f54..dc0f555f8 100644 --- a/tasks/pdp/task_addroot.go +++ b/tasks/pdp/task_addroot.go @@ -2,13 +2,16 @@ package pdp import ( "context" + "errors" "math/big" + "strings" "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ipfs/go-cid" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -47,12 +50,13 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( var addRoots []struct { ID string `db:"id"` PieceCid string `db:"piece_cid"` + PieceCid2 string `db:"piece_cid_v2"` ProofSetID int64 `db:"proof_set_id"` ExtraData []byte `db:"extra_data"` PieceRef string `db:"piece_ref"` } - err = p.db.Select(ctx, &addRoots, `SELECT id, piece_cid, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_root_task_id = $1 AND after_add_root = FALSE`, taskID) + err = p.db.Select(ctx, &addRoots, `SELECT id, piece_cid, piece_cid_v2, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_root_task_id = $1 AND after_add_root = FALSE`, taskID) if err != nil { return false, xerrors.Errorf("failed to select addRoot: %w", err) } @@ -61,7 +65,7 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return false, xerrors.Errorf("no addRoot found for taskID %d", taskID) } - if len(addRoots) > 0 { + if len(addRoots) > 1 { return false, xerrors.Errorf("multiple addRoot found for taskID %d", taskID) } @@ -72,7 +76,12 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return false, xerrors.Errorf("failed to parse piece cid: %w", err) } - pi, err := mk20.GetPieceInfo(pcid) + pcid2, err := cid.Parse(addRoot.PieceCid2) + if err != nil { + return false, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pi, err := mk20.GetPieceInfo(pcid2) if err != nil { return false, xerrors.Errorf("failed to get piece info: %w", err) } @@ -87,7 +96,7 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( rootDataArray := []contract.RootData{ { Root: struct{ Data []byte }{Data: pcid.Bytes()}, - RawSize: new(big.Int).SetUint64(pi.RawSize), + RawSize: new(big.Int).SetUint64(uint64(pi.Size.Unpadded())), }, } @@ -105,8 +114,7 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( Context: ctx, } - pdpContracts := contract.ContractAddresses() - pdpVerifierAddress := pdpContracts.PDPVerifier + pdpVerifierAddress := contract.ContractAddresses().PDPVerifier pdpVerifier, err := contract.NewPDPVerifier(pdpVerifierAddress, p.ethClient) if err != nil { @@ -136,61 +144,24 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return false, xerrors.Errorf("sending transaction: %w", err) } + txHashLower := strings.ToLower(txHash.Hex()) + // Insert into message_waits_eth and pdp_proofset_roots _, err = p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Insert into message_waits_eth _, err = tx.Exec(` INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2) - `, txHash.Hex(), "pending") + `, txHashLower, "pending") if err != nil { return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) } - // Update proof set for initialization upon first add - _, err = tx.Exec(` - UPDATE pdp_proof_sets SET init_ready = true - WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL - `, proofSetID.Uint64()) - if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) - } - - // Insert into pdp_proofset_roots - n, err := tx.Exec(` - INSERT INTO pdp_proofset_root ( - proofset, - piece_cid_v2, - piece_cid, - piece_size, - raw_size, - piece_ref, - add_deal_id, - add_message_hash - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - `, - proofSetID.Uint64(), - pcid.String(), - pi.PieceCIDV1.String(), - pi.Size, - pi.RawSize, - addRoot.PieceRef, - addRoot.ID, - txHash.Hex(), - ) - if err != nil { - return false, xerrors.Errorf("failed to insert into pdp_proofset_root: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("incorrect number of rows inserted for pdp_proofset_root: %d", n) - } - - n, err = tx.Exec(`UPDATE pdp_pipeline SET + n, err := tx.Exec(`UPDATE pdp_pipeline SET after_add_root = TRUE, add_root_task_id = NULL, - add_message_hash = $2, - WHERE add_root_task_id = $1`, taskID, txHash.Hex()) + add_message_hash = $2 + WHERE add_root_task_id = $1`, taskID, txHashLower) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -237,15 +208,19 @@ func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddT WHERE add_root_task_id IS NULL AND after_add_root = FALSE AND after_add_root_msg = FALSE - AND aggregated = TRUE`).Scan(&did) + AND aggregated = TRUE + LIMIT 1`).Scan(&did) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } return false, xerrors.Errorf("failed to query pdp_pipeline: %w", err) } if did == "" { return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_pipeline SET add_root_task_id = $1, WHERE piece_cid = $2 AND after_add_root = FALSE AND after_add_root_msg = FALSE AND aggregated = TRUE`, id, did) + _, err = tx.Exec(`UPDATE pdp_pipeline SET add_root_task_id = $1 WHERE id = $2 AND after_add_root = FALSE AND after_add_root_msg = FALSE AND aggregated = TRUE`, id, did) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -262,3 +237,4 @@ func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddT func (p *PDPTaskAddRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} var _ harmonytask.TaskInterface = &PDPTaskAddRoot{} +var _ = harmonytask.Reg(&PDPTaskAddRoot{}) diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index 414ad6d95..12e3d1356 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -276,13 +276,14 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b } pdp := deal.Products.PDPV1 + retv := deal.Products.RetrievalV1 n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE)`, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12)`, id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting aggregated piece in PDP pipeline: %w", err) } @@ -346,6 +347,10 @@ func (a *AggregatePDPDealTask) schedule(ctx context.Context, taskFunc harmonytas return } + if len(deals) == 0 { + return false, nil + } + deal := deals[0] log.Infow("processing aggregation task", "deal", deal.ID, "count", deal.Count) diff --git a/tasks/pdp/task_delete_root.go b/tasks/pdp/task_delete_root.go index f94abbf36..3e3c69cb2 100644 --- a/tasks/pdp/task_delete_root.go +++ b/tasks/pdp/task_delete_root.go @@ -2,6 +2,7 @@ package pdp import ( "context" + "errors" "math/big" "strings" "time" @@ -9,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -36,7 +38,7 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool ExtraData []byte `db:"extra_data"` } - err = p.db.Select(ctx, &rdeletes, `SELECT id, set_id, roots, extra_data FROM pdp_delete_root WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + err = p.db.Select(ctx, &rdeletes, `SELECT id, set_id, roots, extra_data FROM pdp_root_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) if err != nil { return false, xerrors.Errorf("failed to get task details from DB: %w", err) } @@ -83,6 +85,13 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) } + for i := range roots { + log.Errorf("root: %d", roots[i].Uint64()) + } + log.Errorf("roots: %v", roots) + log.Errorf("proofSetID: %d", proofSetID.Uint64()) + log.Errorf("extraDataBytes: %s", extraDataBytes) + // Pack the method call data data, err := abiData.Pack("scheduleRemovals", proofSetID, roots, extraDataBytes) if err != nil { @@ -93,7 +102,7 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool tx := types.NewTransaction( 0, contract.ContractAddresses().PDPVerifier, - contract.SybilFee(), + big.NewInt(0), 0, nil, data, @@ -108,12 +117,29 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool // Insert into message_waits_eth and pdp_proof_set_delete txHashLower := strings.ToLower(txHash.Hex()) - n, err := p.db.Exec(ctx, `UPDATE pdp_delete_root SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + + comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_root_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_root_delete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_root_delete: %d", n) + } + + _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") + if err != nil { + return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { - return false, xerrors.Errorf("failed to update pdp_delete_root: %w", err) + return false, xerrors.Errorf("failed to commit transaction: %w", err) } - if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_delete_root: %d", n) + + if !comm { + return false, xerrors.Errorf("failed to commit transaction") } return true, nil @@ -145,19 +171,22 @@ func (p *PDPTaskDeleteRoot) schedule(ctx context.Context, taskFunc harmonytask.A stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_delete_root + err := tx.QueryRow(`SELECT id FROM pdp_root_delete WHERE task_id IS NULL - AND tx_hash IS NULL`).Scan(&did) + AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { - return false, xerrors.Errorf("failed to query pdp_delete_root: %w", err) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to query pdp_root_delete: %w", err) } if did == "" { return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_delete_root SET task_id = $1, WHERE id = $2 AND task_id IS NULL AND tx_hash IS NULL`, id, did) + _, err = tx.Exec(`UPDATE pdp_root_delete SET task_id = $1 WHERE id = $2 AND task_id IS NULL AND tx_hash IS NULL`, id, did) if err != nil { - return false, xerrors.Errorf("failed to update pdp_delete_root: %w", err) + return false, xerrors.Errorf("failed to update pdp_root_delete: %w", err) } stop = false // we found a task to schedule, keep going @@ -180,3 +209,4 @@ func NewPDPTaskDeleteRoot(db *harmonydb.DB, sender *message.SenderETH, ethClient } var _ harmonytask.TaskInterface = &PDPTaskDeleteRoot{} +var _ = harmonytask.Reg(&PDPTaskDeleteRoot{}) diff --git a/tasks/pdp/task_delete_rootset.go b/tasks/pdp/task_delete_rootset.go index b9441b937..a647c8b88 100644 --- a/tasks/pdp/task_delete_rootset.go +++ b/tasks/pdp/task_delete_rootset.go @@ -2,6 +2,7 @@ package pdp import ( "context" + "errors" "math/big" "strings" "time" @@ -9,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -43,12 +45,12 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() ExtraData []byte `db:"extra_data"` } - err = p.db.Select(ctx, &pdeletes, `SELECT set_id, extra_data FROM pdp_proof_set_create WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + err = p.db.Select(ctx, &pdeletes, `SELECT set_id, extra_data FROM pdp_proof_set_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) if err != nil { return false, xerrors.Errorf("failed to get task details from DB: %w", err) } - if len(pdeletes) != 0 { + if len(pdeletes) != 1 { return false, xerrors.Errorf("incorrect rows for proofset delete found for taskID %d", taskID) } @@ -97,7 +99,7 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() tx := types.NewTransaction( 0, contract.ContractAddresses().PDPVerifier, - contract.SybilFee(), + big.NewInt(0), 0, nil, data, @@ -112,13 +114,31 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() // Insert into message_waits_eth and pdp_proof_set_delete txHashLower := strings.ToLower(txHash.Hex()) - n, err := p.db.Exec(ctx, `UPDATE pdp_proof_set_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + + comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_proof_set_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_delete: %d", n) + } + + _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") + if err != nil { + return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to commit transaction: %w", err) } - if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_delete: %d", n) + + if !comm { + return false, xerrors.Errorf("failed to commit transaction") } + return true, nil } @@ -129,7 +149,7 @@ func (p *PDPTaskDeleteProofSet) CanAccept(ids []harmonytask.TaskID, engine *harm func (p *PDPTaskDeleteProofSet) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "PDPAddProofSet", + Name: "PDPDelProofSet", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -148,8 +168,11 @@ func (p *PDPTaskDeleteProofSet) schedule(ctx context.Context, taskFunc harmonyta stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_proof_set_delete WHERE task_id IS NULL AND tx_hash IS NULL`).Scan(&id) + err := tx.QueryRow(`SELECT id FROM pdp_proof_set_delete WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } return false, xerrors.Errorf("failed to query pdp_proof_set_delete: %w", err) } if did == "" { @@ -173,3 +196,4 @@ func (p *PDPTaskDeleteProofSet) schedule(ctx context.Context, taskFunc harmonyta func (p *PDPTaskDeleteProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} var _ harmonytask.TaskInterface = &PDPTaskDeleteProofSet{} +var _ = harmonytask.Reg(&PDPTaskDeleteProofSet{}) diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index a6a4ab5f6..2d125beba 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -391,7 +391,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int6 var pieceCid string - err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_proofset_root WHERE proofset = $1 AND root_id = $2`, proofSetID, rootId).Scan(&pieceCid) + err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_proofset_root WHERE proof_set_id = $1 AND root_id = $2`, proofSetID, rootId).Scan(&pieceCid) if err != nil { return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get root and subroot: %w", err) } diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go index 4ac68fa3f..2100ad1fc 100644 --- a/tasks/pdp/task_save_cache.go +++ b/tasks/pdp/task_save_cache.go @@ -2,6 +2,7 @@ package pdp import ( "context" + "errors" "hash" "io" "math" @@ -11,6 +12,7 @@ import ( "github.com/ipfs/go-cid" sha256simd "github.com/minio/sha256-simd" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-padreader" @@ -47,7 +49,7 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) ctx := context.Background() var saveCaches []struct { ID string `db:"id"` - PieceCid string `db:"piece_cid"` + PieceCid string `db:"piece_cid_v2"` ProofSetID int64 `db:"proof_set_id"` PieceRef string `db:"piece_ref"` } @@ -61,7 +63,7 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) return false, xerrors.Errorf("no saveCaches found for taskID %d", taskID) } - if len(saveCaches) > 0 { + if len(saveCaches) > 1 { return false, xerrors.Errorf("multiple saveCaches found for taskID %d", taskID) } @@ -128,7 +130,7 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) } } - n, err := t.db.Exec(ctx, `UPDATE pdp_pipeline SET after_save_cache = TRUE, save_cache_task_id = NULL WHERE save_cache_task_id = $1`, taskID) + n, err := t.db.Exec(ctx, `UPDATE pdp_pipeline SET after_save_cache = TRUE, save_cache_task_id = NULL, indexing_created_at = NOW() WHERE save_cache_task_id = $1`, taskID) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -168,16 +170,19 @@ func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.Ad var did string err := tx.QueryRow(`SELECT id FROM pdp_pipeline WHERE save_cache_task_id IS NULL - AND save_cache_task_id = FALSE - AND aggregated = TRUE`).Scan(&did) + AND after_save_cache = FALSE + AND after_add_root_msg = TRUE`).Scan(&did) if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } return false, xerrors.Errorf("failed to query pdp_pipeline: %w", err) } if did == "" { return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1, WHERE id = $2 AND save_cache_task_id = FALSE AND aggregated = TRUE`, id, did) + _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1 WHERE id = $2 AND after_save_cache = FALSE AND after_add_root_msg = TRUE`, id, did) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -194,6 +199,7 @@ func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.Ad func (t *TaskSavePDPCache) Adder(taskFunc harmonytask.AddTaskFunc) {} var _ harmonytask.TaskInterface = &TaskSavePDPCache{} +var _ = harmonytask.Reg(&TaskSavePDPCache{}) // All the code below is a copy+paste of https://github.com/filecoin-project/go-fil-commp-hashhash/blob/master/commp.go // with modification to output the nodes at a specific height diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index 677ff3719..a2311df3b 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -316,6 +316,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } if !complete { pdp := deal.Products.PDPV1 + retv := deal.Products.RetrievalV1 var newRefID int64 if refIDUsed { err = tx.QueryRow(` @@ -328,10 +329,10 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } @@ -341,10 +342,10 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } else { n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0)`, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type) + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 90ff63734..5318a6774 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -1054,7 +1054,10 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe if err := json.Unmarshal(dbdeal.DDOv1, &dddov1); err != nil { return nil, fmt.Errorf("unmarshal ddov1: %w", err) } - Err = dddov1.Error + if dddov1.Error != "" { + Err.String = dddov1.Error + Err.Valid = true + } } mk20deals[i] = &MK20StorageDeal{ diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index fe71f21b8..cf57d0ff6 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -59,8 +59,8 @@ func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20Storag if err := json.Unmarshal(dbDeal.DDOv1, &dddov1); err != nil { return nil, fmt.Errorf("unmarshal ddov1: %w", err) } - if dddov1.Error.Valid { - ret.Error = dddov1.Error + if dddov1.Error != "" { + ret.Error = sql.NullString{String: dddov1.Error, Valid: true} } } From 99941cf7ad4978fe26d97268d9a8ae2262df7b5d Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 13 Aug 2025 00:36:42 +0400 Subject: [PATCH 24/55] rename PDP --- alertmanager/alerts.go | 4 +- cmd/curio/tasks/tasks.go | 16 +- cmd/sptool/sector.go | 7 +- cmd/sptool/toolbox_deal_client.go | 16 +- deps/deps.go | 6 +- go.mod | 139 +- go.sum | 293 ++- harmony/harmonydb/sql/20240823-ipni.sql | 18 +- .../harmonydb/sql/20250505-market_mk20.sql | 159 +- market/ipni/chunker/serve-chunker.go | 13 +- market/ipni/types/types.go | 16 + market/ipni/types/types_cbor_gen.go | 150 ++ market/mk20/client/client.go | 6 +- market/mk20/http/docs.go | 36 +- market/mk20/http/swagger.json | 36 +- market/mk20/http/swagger.yaml | 44 +- market/mk20/mk20.go | 94 +- market/mk20/mk20_upload.go | 134 +- market/mk20/pdp_v1.go | 99 +- market/mk20/retrieval_v1.go | 3 - pdp/contract/IPDPProvingSchedule.json | 2 +- pdp/contract/PDPVerifier.abi | 163 +- pdp/contract/PDPVerifier.json | 2 +- pdp/contract/README.md | 110 ++ pdp/contract/addresses.go | 2 +- pdp/contract/pdp_verifier.go | 1568 +++++++++-------- pdp/contract/types.go | 6 +- pdp/handlers_upload.go | 2 +- tasks/indexing/task_ipni.go | 4 +- tasks/indexing/task_pdp_ipni.go | 340 ++-- ...eate_watch.go => data_set_create_watch.go} | 80 +- ...lete_watch.go => data_set_delete_watch.go} | 50 +- ...ot_watch.go => dataset_add_piece_watch.go} | 126 +- ..._watch.go => dataset_delete_root_watch.go} | 50 +- ...k_add_proofset.go => task_add_data_set.go} | 52 +- .../{task_addroot.go => task_add_piece.go} | 84 +- tasks/pdp/task_aggregation.go | 10 +- ...ete_rootset.go => task_delete_data_set.go} | 52 +- ...sk_delete_root.go => task_delete_piece.go} | 70 +- tasks/pdp/task_init_pp.go | 66 +- tasks/pdp/task_next_pp.go | 57 +- tasks/pdp/task_prove.go | 151 +- tasks/pdp/task_save_cache.go | 38 +- tasks/piece/task_aggregate_chunks.go | 50 +- tasks/seal/task_submit_commit.go | 13 +- tasks/storage-market/mk20.go | 272 ++- tasks/storage-market/storage_market.go | 1 + web/api/webrpc/pdp.go | 12 +- 48 files changed, 2837 insertions(+), 1885 deletions(-) create mode 100644 market/ipni/types/types.go create mode 100644 market/ipni/types/types_cbor_gen.go create mode 100644 pdp/contract/README.md rename tasks/pdp/{proofset_create_watch.go => data_set_create_watch.go} (65%) rename tasks/pdp/{proofset_delete_watch.go => data_set_delete_watch.go} (72%) rename tasks/pdp/{proofset_addroot_watch.go => dataset_add_piece_watch.go} (62%) rename tasks/pdp/{proofset_delete_root_watch.go => dataset_delete_root_watch.go} (68%) rename tasks/pdp/{task_add_proofset.go => task_add_data_set.go} (69%) rename tasks/pdp/{task_addroot.go => task_add_piece.go} (65%) rename tasks/pdp/{task_delete_rootset.go => task_delete_data_set.go} (67%) rename tasks/pdp/{task_delete_root.go => task_delete_piece.go} (63%) diff --git a/alertmanager/alerts.go b/alertmanager/alerts.go index 65f12df6e..195d1e7ab 100644 --- a/alertmanager/alerts.go +++ b/alertmanager/alerts.go @@ -3,7 +3,6 @@ package alertmanager import ( "bytes" "context" - "database/sql" "fmt" "math" "strings" @@ -13,6 +12,7 @@ import ( "github.com/dustin/go-humanize" cbor "github.com/ipfs/go-ipld-cbor" "github.com/samber/lo" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -346,7 +346,7 @@ func (al *alerts) getAddresses() ([]address.Address, []address.Address, error) { cfg := config.DefaultCurioConfig() err := al.db.QueryRow(al.ctx, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text) if err != nil { - if strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + if strings.Contains(err.Error(), pgx.ErrNoRows.Error()) { return nil, nil, xerrors.Errorf("missing layer '%s' ", layer) } return nil, nil, xerrors.Errorf("could not read layer '%s': %w", layer, err) diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index da4e2e9c8..0dac09c53 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -286,23 +286,23 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan es := getSenderEth() sdeps.EthSender = es - pdp.NewWatcherCreate(db, must.One(dependencies.EthClient.Val()), chainSched) - pdp.NewWatcherRootAdd(db, chainSched) + pdp.NewWatcherDataSetCreate(db, must.One(dependencies.EthClient.Val()), chainSched) + pdp.NewWatcherPieceAdd(db, chainSched) pdp.NewWatcherDelete(db, chainSched) - pdp.NewWatcherRootDelete(db, chainSched) + pdp.NewWatcherPieceDelete(db, chainSched) pdpProveTask := pdp.NewProveTask(chainSched, db, must.One(dependencies.EthClient.Val()), dependencies.Chain, es, dependencies.CachedPieceReader, iStore) pdpNextProvingPeriodTask := pdp.NewNextProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpInitProvingPeriodTask := pdp.NewInitProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) pdpNotifTask := pdp.NewPDPNotifyTask(db) - addProofSetTask := pdp.NewPDPTaskAddProofSet(db, es, must.One(dependencies.EthClient.Val()), full) - pdpAddRoot := pdp.NewPDPTaskAddRoot(db, es, must.One(dependencies.EthClient.Val())) - pdpDelRoot := pdp.NewPDPTaskDeleteRoot(db, es, must.One(dependencies.EthClient.Val())) - pdpDelProofSetTask := pdp.NewPDPTaskDeleteProofSet(db, es, must.One(dependencies.EthClient.Val()), full) + addProofSetTask := pdp.NewPDPTaskAddDataSet(db, es, must.One(dependencies.EthClient.Val()), full) + pdpAddRoot := pdp.NewPDPTaskAddPiece(db, es, must.One(dependencies.EthClient.Val())) + pdpDelRoot := pdp.NewPDPTaskDeletePiece(db, es, must.One(dependencies.EthClient.Val())) + pdpDelProofSetTask := pdp.NewPDPTaskDeleteDataSet(db, es, must.One(dependencies.EthClient.Val()), full) pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) - pdpCache := pdp.NewTaskSavePDPCache(db, dependencies.CachedPieceReader, iStore) + pdpCache := pdp.NewTaskPDPSaveCache(db, dependencies.CachedPieceReader, iStore) activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, pdpAddRoot, addProofSetTask, pdpAggregateTask, pdpCache, pdpDelRoot, pdpDelProofSetTask) } diff --git a/cmd/sptool/sector.go b/cmd/sptool/sector.go index 2ede0d58c..0b22efded 100644 --- a/cmd/sptool/sector.go +++ b/cmd/sptool/sector.go @@ -710,11 +710,6 @@ Extensions will be clamped at either the maximum sector extension of 3.5 years/1 return err } - declMax, err := policy.GetDeclarationsMax(nv) - if err != nil { - return err - } - addrSectors := sectorsMax if cctx.Int("max-sectors") != 0 { addrSectors = cctx.Int("max-sectors") @@ -794,7 +789,7 @@ Extensions will be clamped at either the maximum sector extension of 3.5 years/1 sectorsInDecl := int(sectorsWithoutClaimsCount) + len(sectorsWithClaims) scount += sectorsInDecl - if scount > addrSectors || len(p.Extensions) >= declMax { + if scount > addrSectors || len(p.Extensions) >= policy.DeclarationsMax { params = append(params, p) p = miner.ExtendSectorExpiration2Params{} scount = sectorsInDecl diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 04347731d..320df54c2 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -2540,9 +2540,9 @@ var mk20PDPDealCmd = &cli.Command{ if !proofSetSet { return xerrors.Errorf("proofset-id must be set when adding a root") } - pdp.AddRoot = true + pdp.AddPiece = true pdp.RecordKeeper = recordKeeper - pdp.ProofSetID = &proofsetID + pdp.DataSetID = &proofsetID pdp.ExtraData = extraBytes ret = &mk20.RetrievalV1{ Indexing: true, @@ -2554,17 +2554,17 @@ var mk20PDPDealCmd = &cli.Command{ if !proofSetSet { return xerrors.Errorf("proofset-id must be set when removing a root") } - pdp.DeleteRoot = true + pdp.DeletePiece = true pdp.RecordKeeper = recordKeeper - pdp.ProofSetID = &proofsetID - pdp.RootIDs = rootIDs + pdp.DataSetID = &proofsetID + pdp.PieceIDs = rootIDs pdp.ExtraData = extraBytes d = nil } if addProofset { pdp.RecordKeeper = recordKeeper - pdp.CreateProofSet = true + pdp.CreateDataSet = true pdp.ExtraData = extraBytes d = nil } @@ -2574,8 +2574,8 @@ var mk20PDPDealCmd = &cli.Command{ return xerrors.Errorf("proofset-id must be set when deleting proof-set") } pdp.RecordKeeper = recordKeeper - pdp.DeleteProofSet = true - pdp.ProofSetID = &proofsetID + pdp.DeleteDataSet = true + pdp.DataSetID = &proofsetID pdp.ExtraData = extraBytes d = nil } diff --git a/deps/deps.go b/deps/deps.go index 3ff89d2bc..3eceacdc9 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "crypto/rand" - "database/sql" "encoding/base64" "errors" "fmt" @@ -23,6 +22,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/samber/lo" "github.com/urfave/cli/v2" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" @@ -426,7 +426,7 @@ func GetConfig(ctx context.Context, layers []string, db *harmonydb.DB) (*config. text := "" err := db.QueryRow(ctx, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text) if err != nil { - if strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + if strings.Contains(err.Error(), pgx.ErrNoRows.Error()) { return nil, fmt.Errorf("missing layer '%s' ", layer) } if layer == "base" { @@ -458,7 +458,7 @@ func updateBaseLayer(ctx context.Context, db *harmonydb.DB) error { text := "" err = tx.QueryRow(`SELECT config FROM harmony_config WHERE title=$1`, "base").Scan(&text) if err != nil { - if strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + if strings.Contains(err.Error(), pgx.ErrNoRows.Error()) { return false, fmt.Errorf("missing layer 'base' ") } return false, fmt.Errorf("could not read layer 'base': %w", err) diff --git a/go.mod b/go.mod index beb66115e..891f9da05 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/curio -go 1.23.7 +go 1.23.10 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 @@ -18,7 +18,7 @@ require ( github.com/etclabscore/go-openrpc-reflect v0.0.36 github.com/ethereum/go-ethereum v1.14.13 github.com/fatih/color v1.18.0 - github.com/filecoin-project/filecoin-ffi v1.32.0 + github.com/filecoin-project/filecoin-ffi v1.33.1-dev github.com/filecoin-project/go-address v1.2.0 github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.2 @@ -26,14 +26,14 @@ require ( github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-data-segment v0.0.1 - github.com/filecoin-project/go-f3 v0.8.4 + github.com/filecoin-project/go-f3 v0.8.9 github.com/filecoin-project/go-fil-commcid v0.2.0 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 github.com/filecoin-project/go-jsonrpc v0.7.1 github.com/filecoin-project/go-padreader v0.0.1 - github.com/filecoin-project/go-state-types v0.16.0 + github.com/filecoin-project/go-state-types v0.17.0-dev2 github.com/filecoin-project/go-statestore v0.2.0 - github.com/filecoin-project/lotus v1.33.0 + github.com/filecoin-project/lotus v1.33.1 github.com/filecoin-project/specs-actors/v2 v2.3.6 github.com/filecoin-project/specs-actors/v5 v5.0.6 github.com/filecoin-project/specs-actors/v6 v6.0.2 @@ -44,7 +44,7 @@ require ( github.com/go-chi/httprate v0.15.0 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.6.0 + github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.1 @@ -55,41 +55,41 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/icza/backscanner v0.0.0-20240328210400-b40c3a86dec5 github.com/invopop/jsonschema v0.12.0 - github.com/ipfs/boxo v0.24.3 - github.com/ipfs/go-block-format v0.2.0 + github.com/ipfs/boxo v0.33.0 + github.com/ipfs/go-block-format v0.2.2 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-cidutil v0.1.0 - github.com/ipfs/go-datastore v0.6.0 + github.com/ipfs/go-datastore v0.8.2 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-graphsync v0.17.0 - github.com/ipfs/go-ipld-cbor v0.2.0 - github.com/ipfs/go-ipld-format v0.6.0 - github.com/ipfs/go-log/v2 v2.5.1 + github.com/ipfs/go-ipld-cbor v0.2.1 + github.com/ipfs/go-ipld-format v0.6.2 + github.com/ipfs/go-log/v2 v2.6.0 github.com/ipld/frisbii v0.6.1 github.com/ipld/go-car v0.6.2 - github.com/ipld/go-car/v2 v2.14.2 + github.com/ipld/go-car/v2 v2.14.3 github.com/ipld/go-ipld-prime v0.21.0 - github.com/ipni/go-libipni v0.6.13 + github.com/ipni/go-libipni v0.6.19 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jellydator/ttlcache/v2 v2.11.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/libp2p/go-buffer-pool v0.1.0 - github.com/libp2p/go-libp2p v0.39.1 + github.com/libp2p/go-libp2p v0.42.0 github.com/manifoldco/promptui v0.9.0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.14.0 + github.com/multiformats/go-multiaddr v0.16.0 github.com/multiformats/go-multibase v0.2.0 - github.com/multiformats/go-multicodec v0.9.1 + github.com/multiformats/go-multicodec v0.9.2 github.com/multiformats/go-multihash v0.2.3 github.com/multiformats/go-varint v0.0.7 github.com/oklog/ulid v1.3.1 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.21.1 + github.com/prometheus/client_golang v1.22.0 github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/samber/lo v1.47.0 @@ -107,14 +107,14 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.36.0 - golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa - golang.org/x/net v0.38.0 - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 - golang.org/x/term v0.30.0 - golang.org/x/text v0.23.0 - golang.org/x/tools v0.31.0 + golang.org/x/crypto v0.39.0 + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b + golang.org/x/net v0.41.0 + golang.org/x/sync v0.15.0 + golang.org/x/sys v0.33.0 + golang.org/x/term v0.32.0 + golang.org/x/text v0.26.0 + golang.org/x/tools v0.34.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da ) @@ -175,7 +175,7 @@ require ( github.com/filecoin-project/go-crypto v0.1.0 // indirect github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect - github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 // indirect + github.com/filecoin-project/go-hamt-ipld/v3 v3.4.1 // indirect github.com/filecoin-project/go-paramfetch v0.0.4 // indirect github.com/filecoin-project/go-statemachine v1.0.3 // indirect github.com/filecoin-project/go-storedcounter v0.1.0 // indirect @@ -187,6 +187,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gammazero/chanqueue v1.1.0 // indirect github.com/gammazero/deque v1.0.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/gdamore/tcell/v2 v2.2.0 // indirect @@ -207,7 +208,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect + github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hannahhoward/go-pubsub v1.0.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -229,14 +230,14 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect - github.com/ipfs/go-ipld-legacy v0.2.1 // indirect + github.com/ipfs/go-ipld-legacy v0.2.2 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-merkledag v0.11.0 // indirect - github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/ipfs/go-metrics-interface v0.3.0 // indirect github.com/ipfs/go-peertaskqueue v0.8.2 // indirect - github.com/ipfs/go-unixfsnode v1.9.2 // indirect + github.com/ipfs/go-unixfsnode v1.10.1 // indirect github.com/ipfs/go-verifcid v0.0.3 // indirect - github.com/ipld/go-codec-dagpb v1.6.0 // indirect + github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-adl-hamt v0.0.0-20240322071803-376decb85801 // indirect github.com/ipld/go-trustless-utils v0.4.1 // indirect github.com/ipni/index-provider v0.15.4 // indirect @@ -245,7 +246,6 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect - github.com/jbenet/goprocess v0.1.4 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -253,21 +253,20 @@ require ( github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/koron/go-ssdp v0.0.5 // indirect + github.com/koron/go-ssdp v0.0.6 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect - github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect - github.com/libp2p/go-libp2p-pubsub v0.13.0 // indirect - github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.33.1 // indirect + github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect + github.com/libp2p/go-libp2p-pubsub v0.14.1 // indirect + github.com/libp2p/go-libp2p-record v0.3.1 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect github.com/libp2p/go-maddr-filter v0.1.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.2 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect - github.com/libp2p/go-yamux/v4 v4.0.2 // indirect + github.com/libp2p/go-yamux/v5 v5.0.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -276,7 +275,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.16 // indirect - github.com/miekg/dns v1.1.63 // indirect + github.com/miekg/dns v1.1.66 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect @@ -286,45 +285,42 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multistream v0.6.0 // indirect + github.com/multiformats/go-multistream v0.6.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nikkolasg/hexjson v0.1.0 // indirect github.com/nkovacs/streamquote v1.1.0 // indirect - github.com/onsi/ginkgo/v2 v2.22.2 // indirect + github.com/onsi/ginkgo/v2 v2.23.4 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pion/datachannel v1.5.10 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect - github.com/pion/dtls/v3 v3.0.4 // indirect - github.com/pion/ice/v2 v2.3.37 // indirect - github.com/pion/ice/v4 v4.0.8 // indirect - github.com/pion/interceptor v0.1.39 // indirect + github.com/pion/dtls/v3 v3.0.6 // indirect + github.com/pion/ice/v4 v4.0.10 // indirect + github.com/pion/interceptor v0.1.40 // indirect github.com/pion/logging v0.2.3 // indirect - github.com/pion/mdns v0.0.12 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.15 // indirect - github.com/pion/rtp v1.8.18 // indirect - github.com/pion/sctp v1.8.37 // indirect - github.com/pion/sdp/v3 v3.0.10 // indirect - github.com/pion/srtp/v3 v3.0.4 // indirect + github.com/pion/rtp v1.8.19 // indirect + github.com/pion/sctp v1.8.39 // indirect + github.com/pion/sdp/v3 v3.0.13 // indirect + github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/transport/v3 v3.0.7 // indirect - github.com/pion/turn/v2 v2.1.6 // indirect - github.com/pion/turn/v4 v4.0.0 // indirect - github.com/pion/webrtc/v4 v4.0.10 // indirect + github.com/pion/turn/v4 v4.0.2 // indirect + github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.64.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.50.1 // indirect + github.com/quic-go/quic-go v0.52.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -358,32 +354,33 @@ require ( go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v1.28.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/sdk v1.34.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/dig v1.18.0 // indirect - go.uber.org/fx v1.23.0 // indirect - go.uber.org/mock v0.5.0 // indirect + go.uber.org/automaxprocs v1.6.0 // indirect + go.uber.org/dig v1.19.0 // indirect + go.uber.org/fx v1.24.0 // indirect + go.uber.org/mock v0.5.2 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/time v0.11.0 // indirect - gonum.org/v1/gonum v0.15.0 // indirect + golang.org/x/mod v0.25.0 // indirect + golang.org/x/time v0.12.0 // indirect + gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250212204824-5a70512c5d8b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b // indirect google.golang.org/grpc v1.70.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect - lukechampine.com/blake3 v1.4.0 // indirect + lukechampine.com/blake3 v1.4.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 5e37799f9..06474705e 100644 --- a/go.sum +++ b/go.sum @@ -103,7 +103,6 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -324,8 +323,8 @@ github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE= github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w= github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= -github.com/filecoin-project/go-f3 v0.8.4 h1:qbdsiMYPWkM2zR/8oFDl4VvHm2YTF7xnr5m/smYKanA= -github.com/filecoin-project/go-f3 v0.8.4/go.mod h1:k23EMAx090NIWKlAYuO4TfjmfQTlIovaQ0nns960s9M= +github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM= +github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI= github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= @@ -336,8 +335,8 @@ github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxl github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= -github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 h1:nYs6OPUF8KbZ3E8o9p9HJnQaE8iugjHR5WYVMcicDJc= -github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0/go.mod h1:s0qiHRhFyrgW0SvdQMSJFQxNa4xEIG5XvqCBZUEgcbc= +github.com/filecoin-project/go-hamt-ipld/v3 v3.4.1 h1:wl+ZHruCcE9LvwU7blpwWn35XOcRS6+IBg75G7ZzxzY= +github.com/filecoin-project/go-hamt-ipld/v3 v3.4.1/go.mod h1:AqjryNfkxffpnqsa5mwnJHlazhVqF6W2nilu+VYKIq8= github.com/filecoin-project/go-jsonrpc v0.7.1 h1:++oUd7R3aYibLKXS/DsO348Lco+1cJbfCwRiv8awHFQ= github.com/filecoin-project/go-jsonrpc v0.7.1/go.mod h1:lAUpS8BSVtKaA8+/CFUMA5dokMiSM7n0ehf8bHOFdpE= github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs= @@ -351,8 +350,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= -github.com/filecoin-project/go-state-types v0.16.0/go.mod h1:YCESyrqnyu17y0MazbV6Uwma5+BrMvEKEQp5QWeIf9g= +github.com/filecoin-project/go-state-types v0.17.0-dev2 h1:2P7UxGmjmo8dTHtC6mn7I1/HYd6MNPgMMt4jTQ/Juds= +github.com/filecoin-project/go-state-types v0.17.0-dev2/go.mod h1:em4yo9mglrdyHbcsxelHCSKMjLdJLddLERWQe6J8vYc= github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk= github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= @@ -360,8 +359,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus= github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= -github.com/filecoin-project/lotus v1.33.0 h1:aAzAo/sC2lVZfX9/erTMhf+g8bZPoJGfW/4w2gIsPno= -github.com/filecoin-project/lotus v1.33.0/go.mod h1:99mKdjsAA5znxyxsYawW5J4UnjnS/cNQrrKznNAuxcs= +github.com/filecoin-project/lotus v1.33.1 h1:eMZ1DBwnJm7BT++psUEDi0XxNRzpUK8qGiNYBCbBYZE= +github.com/filecoin-project/lotus v1.33.1/go.mod h1:70y4vylPpIetPXr+GMF88ssSPG9X5YCh1Ev1PkxEQzc= github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM= github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -393,8 +392,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gammazero/channelqueue v0.2.2 h1:ufNzIbeDBxNfHj0m5uwUfOwvTmHF/O40hu2ZNnvF+/8= -github.com/gammazero/channelqueue v0.2.2/go.mod h1:824o5HHE+yO1xokh36BIuSv8YWwXW0364ku91eRMFS4= +github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw= +github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/gbrlsnchs/jwt/v3 v3.0.1 h1:lbUmgAKpxnClrKloyIwpxm4OuWeDl5wLk52G91ODPw4= @@ -533,8 +532,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -551,13 +550,12 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc= -github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= +github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -645,8 +643,8 @@ github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uO github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c= -github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg= +github.com/ipfs/boxo v0.33.0 h1:9ow3chwkDzMj0Deq4AWRUEI7WnIIV7SZhPTzzG2mmfw= +github.com/ipfs/boxo v0.33.0/go.mod h1:3IPh7YFcCIcKp6o02mCHovrPntoT5Pctj/7j4syh/RM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= @@ -656,8 +654,8 @@ github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niY github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= -github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= +github.com/ipfs/go-block-format v0.2.2 h1:uecCTgRwDIXyZPgYspaLXoMiMmxQpSx2aq34eNc4YvQ= +github.com/ipfs/go-block-format v0.2.2/go.mod h1:vmuefuWU6b+9kIU0vZJgpiJt1yicQz9baHXE8qR+KB8= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= @@ -678,8 +676,8 @@ github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= -github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-datastore v0.8.2 h1:Jy3wjqQR6sg/LhyY0NIePZC3Vux19nLtg7dx0TVqr6U= +github.com/ipfs/go-datastore v0.8.2/go.mod h1:W+pI1NsUsz3tcsAACMtfC+IZdnQTnC/7VfPoJBQuts0= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -729,14 +727,14 @@ github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= -github.com/ipfs/go-ipld-cbor v0.2.0 h1:VHIW3HVIjcMd8m4ZLZbrYpwjzqlVUfjLM7oK4T5/YF0= -github.com/ipfs/go-ipld-cbor v0.2.0/go.mod h1:Cp8T7w1NKcu4AQJLqK0tWpd1nkgTxEVB5C6kVpLW6/0= +github.com/ipfs/go-ipld-cbor v0.2.1 h1:H05yEJbK/hxg0uf2AJhyerBDbjOuHX4yi+1U/ogRa7E= +github.com/ipfs/go-ipld-cbor v0.2.1/go.mod h1:x9Zbeq8CoE5R2WicYgBMcr/9mnkQ0lHddYWJP2sMV3A= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= -github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= -github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= -github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= +github.com/ipfs/go-ipld-format v0.6.2 h1:bPZQ+A05ol0b3lsJSl0bLvwbuQ+HQbSsdGTy4xtYUkU= +github.com/ipfs/go-ipld-format v0.6.2/go.mod h1:nni2xFdHKx5lxvXJ6brt/pndtGxKAE+FPR1rg4jTkyk= +github.com/ipfs/go-ipld-legacy v0.2.2 h1:DThbqCPVLpWBcGtU23KDLiY2YRZZnTkXQyfz8aOfBkQ= +github.com/ipfs/go-ipld-legacy v0.2.2/go.mod h1:hhkj+b3kG9b2BcUNw8IFYAsfeNo8E3U7eYlWeAOPyDU= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= @@ -747,22 +745,23 @@ github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscw github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= -github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= -github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg= +github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= -github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= +github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= github.com/ipfs/go-peertaskqueue v0.8.2/go.mod h1:L6QPvou0346c2qPJNiJa6BvOibxDfaiPlqHInmzg0FA= -github.com/ipfs/go-test v0.0.4 h1:DKT66T6GBB6PsDFLoO56QZPrOmzJkqU1FZH5C9ySkew= -github.com/ipfs/go-test v0.0.4/go.mod h1:qhIM1EluEfElKKM6fnWxGn822/z9knUGM1+I/OAQNKI= +github.com/ipfs/go-test v0.2.2 h1:1yjYyfbdt1w93lVzde6JZ2einh3DIV40at4rVoyEcE8= +github.com/ipfs/go-test v0.2.2/go.mod h1:cmLisgVwkdRCnKu/CFZOk2DdhOcwghr5GsHeqwexoRA= github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= -github.com/ipfs/go-unixfsnode v1.9.2 h1:0A12BYs4XOtDPJTMlwmNPlllDfqcc4yie4e919hcUXk= -github.com/ipfs/go-unixfsnode v1.9.2/go.mod h1:v1nuMFHf4QTIhFUdPMvg1nQu7AqDLvIdwyvJ531Ot1U= +github.com/ipfs/go-unixfsnode v1.10.1 h1:hGKhzuH6NSzZ4y621wGuDspkjXRNG3B+HqhlyTjSwSM= +github.com/ipfs/go-unixfsnode v1.10.1/go.mod h1:eguv/otvacjmfSbYvmamc9ssNAzLvRk0+YN30EYeOOY= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= @@ -771,10 +770,10 @@ github.com/ipld/frisbii v0.6.1/go.mod h1:5alsRVbOyUbZ2In70AdJ4VOLh13LkmAMUomotJa github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= -github.com/ipld/go-car/v2 v2.14.2 h1:9ERr7KXpCC7If0rChZLhYDlyr6Bes6yRKPJnCO3hdHY= -github.com/ipld/go-car/v2 v2.14.2/go.mod h1:0iPB/825lTZLU2zPK5bVTk/R3V2612E1VI279OGSXWA= -github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= -github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-car/v2 v2.14.3 h1:1Mhl82/ny8MVP+w1M4LXbj4j99oK3gnuZG2GmG1IhC8= +github.com/ipld/go-car/v2 v2.14.3/go.mod h1:/vpSvPngOX8UnvmdFJ3o/mDgXa9LuyXsn7wxOzHDYQE= +github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= +github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-fixtureplate v0.0.3 h1:Qb/rBBnYP8IiK+VLq89y2NPZ3iQeQpAi9YK3oSleVGs= github.com/ipld/go-fixtureplate v0.0.3/go.mod h1:i97D4Kbzvvk3OxGfwgKL3fpmzxXhpLEVpQ0mT4ek24s= github.com/ipld/go-ipld-adl-hamt v0.0.0-20240322071803-376decb85801 h1:B5P5TdYpNt0ZEbbZ4Tjj7mO3dWENbT5PxOJ3xgj+lnQ= @@ -789,8 +788,8 @@ github.com/ipld/go-trustless-utils v0.4.1 h1:puA14381Hg2LzH724mZ5ZFKFx+FFjjT5fPF github.com/ipld/go-trustless-utils v0.4.1/go.mod h1:DgGuyfJ33goYwYVisjnxrlra0HVmZuHWVisVIkzVo1o= github.com/ipld/ipld/specs v0.0.0-20231012031213-54d3b21deda4 h1:0VXv637/xpI0Pb5J8K+K8iRtTw4DOcxs0MB1HMzfwNY= github.com/ipld/ipld/specs v0.0.0-20231012031213-54d3b21deda4/go.mod h1:WcT0DfRe+e2QFY0kcbsOnuT6jL5Q0JNZ83I5DHIdStg= -github.com/ipni/go-libipni v0.6.13 h1:6fQU6ZFu8fi0DZIs4VXZrIFbT9r97dNmNl7flWMVblE= -github.com/ipni/go-libipni v0.6.13/go.mod h1:+hNohg7Tx8ML2a/Ei19zUxCnSqtqXiHySlqHIwPhQyQ= +github.com/ipni/go-libipni v0.6.19 h1:f19SYd585pqzX5C6M8vFP1veL7fVYuBELIFfsMjOMZQ= +github.com/ipni/go-libipni v0.6.19/go.mod h1:pu+1iqsmN6TE2JcHjEKnDkLkONy1PW0rM4qjeF3jnHM= github.com/ipni/index-provider v0.15.4 h1:K64q94r6M/QFyIvRwMxC6oOv92cOixCzy/awGmsBEXI= github.com/ipni/index-provider v0.15.4/go.mod h1:R08LoUrA12fiqtDVUwLAv+g09BPY0FsCG58JvFEyVzo= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= @@ -868,8 +867,8 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.5 h1:E1iSMxIs4WqxTbIBLtmNBeOOC+1sCIXQeqTWVnpmwhk= -github.com/koron/go-ssdp v0.0.5/go.mod h1:Qm59B7hpKpDqfyRNWRNr00jGwLdXjDyZh6y7rH6VS0w= +github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= +github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -900,8 +899,8 @@ github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= -github.com/libp2p/go-libp2p v0.39.1 h1:1Ur6rPCf3GR+g8jkrnaQaM0ha2IGespsnNlCqJLLALE= -github.com/libp2p/go-libp2p v0.39.1/go.mod h1:3zicI8Lp7Isun+Afo/JOACUbbJqqR2owK6RQWFsVAbI= +github.com/libp2p/go-libp2p v0.42.0 h1:A8foZk+ZEhZTv0Jb++7xUFlrFhBDv4j2Vh/uq4YX+KE= +github.com/libp2p/go-libp2p v0.42.0/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= @@ -912,10 +911,10 @@ github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= -github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= -github.com/libp2p/go-libp2p-kad-dht v0.28.1/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= -github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= -github.com/libp2p/go-libp2p-kbucket v0.6.4/go.mod h1:jp6w82sczYaBsAypt5ayACcRJi0lgsba7o4TzJKEfWA= +github.com/libp2p/go-libp2p-kad-dht v0.33.1 h1:hKFhHMf7WH69LDjaxsJUWOU6qZm71uO47M/a5ijkiP0= +github.com/libp2p/go-libp2p-kad-dht v0.33.1/go.mod h1:CdmNk4VeGJa9EXM9SLNyNVySEvduKvb+5rSC/H4pLAo= +github.com/libp2p/go-libp2p-kbucket v0.7.0 h1:vYDvRjkyJPeWunQXqcW2Z6E93Ywx7fX0jgzb/dGOKCs= +github.com/libp2p/go-libp2p-kbucket v0.7.0/go.mod h1:blOINGIj1yiPYlVEX0Rj9QwEkmVnz3EP8LK1dRKBC6g= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -923,13 +922,13 @@ github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCv github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-pubsub v0.13.0 h1:RmFQ2XAy3zQtbt2iNPy7Tt0/3fwTnHpCQSSnmGnt1Ps= -github.com/libp2p/go-libp2p-pubsub v0.13.0/go.mod h1:m0gpUOyrXKXdE7c8FNQ9/HLfWbxaEw7xku45w+PaqZo= +github.com/libp2p/go-libp2p-pubsub v0.14.1 h1:XK/rPKZKhPvRrtsjvfwrOZPnQQbGLmaEg7u6qnJfn8U= +github.com/libp2p/go-libp2p-pubsub v0.14.1/go.mod h1:MKPU5vMI8RRFyTP0HfdsF9cLmL1nHAeJm44AxJGJx44= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= -github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= -github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= -github.com/libp2p/go-libp2p-routing-helpers v0.7.4 h1:6LqS1Bzn5CfDJ4tzvP9uwh42IB7TJLNFJA6dEeGBv84= -github.com/libp2p/go-libp2p-routing-helpers v0.7.4/go.mod h1:we5WDj9tbolBXOuF1hGOkR+r7Uh1408tQbAKaT5n1LE= +github.com/libp2p/go-libp2p-record v0.3.1 h1:cly48Xi5GjNw5Wq+7gmjfBiG9HCzQVkiZOUZ8kUl+Fg= +github.com/libp2p/go-libp2p-record v0.3.1/go.mod h1:T8itUkLcWQLCYMqtX7Th6r7SexyUJpIyPgks757td/E= +github.com/libp2p/go-libp2p-routing-helpers v0.7.5 h1:HdwZj9NKovMx0vqq6YNPTh6aaNzey5zHD7HeLJtq6fI= +github.com/libp2p/go-libp2p-routing-helpers v0.7.5/go.mod h1:3YaxrwP0OBPDD7my3D0KxfR89FlcX/IEbxDEDfAmj98= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -950,8 +949,6 @@ github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= -github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= -github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFPuZ8= github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= @@ -965,8 +962,8 @@ github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjV github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= -github.com/libp2p/go-yamux/v4 v4.0.2 h1:nrLh89LN/LEiqcFiqdKDRHjGstN300C1269K/EX0CPU= -github.com/libp2p/go-yamux/v4 v4.0.2/go.mod h1:C808cCRgOs1iBwY4S71T5oxgMxgLmqUw56qh4AeBW2o= +github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= +github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= @@ -998,7 +995,6 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1011,8 +1007,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= -github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= +github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1066,8 +1062,8 @@ github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= -github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= +github.com/multiformats/go-multiaddr v0.16.0 h1:oGWEVKioVQcdIOBlYM8BH1rZDWOGJSqr9/BKl6zQ4qc= +github.com/multiformats/go-multiaddr v0.16.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= @@ -1080,8 +1076,8 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo= -github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= +github.com/multiformats/go-multicodec v0.9.2 h1:YrlXCuqxjqm3bXl+vBq5LKz5pz4mvAsugdqy78k0pXQ= +github.com/multiformats/go-multicodec v0.9.2/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1092,8 +1088,8 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= -github.com/multiformats/go-multistream v0.6.0 h1:ZaHKbsL404720283o4c/IHQXiS6gb8qAN5EIJ4PN5EA= -github.com/multiformats/go-multistream v0.6.0/go.mod h1:MOyoG5otO24cHIg8kf9QW2/NozURlkP/rvi2FQJyCPg= +github.com/multiformats/go-multistream v0.6.1 h1:4aoX5v6T+yWmc2raBHsTvzmFhOI8WVOer28DeBBEYdQ= +github.com/multiformats/go-multistream v0.6.1/go.mod h1:ksQf6kqHAb6zIsyw7Zm+gAuVo57Qbq84E27YlYqavqw= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= @@ -1110,9 +1106,8 @@ github.com/nikkolasg/hexjson v0.1.0/go.mod h1:fbGbWFZ0FmJMFbpCMtJpwb0tudVxSSZ+Es github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/nkovacs/streamquote v1.1.0 h1:wDY1+Hikdx4iOmZZBFLXvwLr7zj9uPIoXfijz+6ad2g= github.com/nkovacs/streamquote v1.1.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= -github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -1124,14 +1119,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= -github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= +github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1155,33 +1150,29 @@ github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oL github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= -github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U= -github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg= -github.com/pion/ice/v2 v2.3.37 h1:ObIdaNDu1rCo7hObhs34YSBcO7fjslJMZV0ux+uZWh0= -github.com/pion/ice/v2 v2.3.37/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ= -github.com/pion/ice/v4 v4.0.8 h1:ajNx0idNG+S+v9Phu4LSn2cs8JEfTsA1/tEjkkAVpFY= -github.com/pion/ice/v4 v4.0.8/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= -github.com/pion/interceptor v0.1.39 h1:Y6k0bN9Y3Lg/Wb21JBWp480tohtns8ybJ037AGr9UuA= -github.com/pion/interceptor v0.1.39/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= +github.com/pion/dtls/v3 v3.0.6 h1:7Hkd8WhAJNbRgq9RgdNh1aaWlZlGpYTzdqjy9x9sK2E= +github.com/pion/dtls/v3 v3.0.6/go.mod h1:iJxNQ3Uhn1NZWOMWlLxEEHAN5yX7GyPvvKw04v9bzYU= +github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4= +github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= +github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= +github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= -github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= -github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.18 h1:yEAb4+4a8nkPCecWzQB6V/uEU18X1lQCGAQCjP+pyvU= -github.com/pion/rtp v1.8.18/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= -github.com/pion/sctp v1.8.37 h1:ZDmGPtRPX9mKCiVXtMbTWybFw3z/hVKAZgU81wcOrqs= -github.com/pion/sctp v1.8.37/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= -github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA= -github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= -github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= -github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= +github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= +github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= +github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= +github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= +github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= +github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= +github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= @@ -1190,16 +1181,12 @@ github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1A github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q= github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= -github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= -github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= -github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= -github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= -github.com/pion/webrtc/v4 v4.0.10 h1:Hq/JLjhqLxi+NmCtE8lnRPDr8H4LcNvwg8OxVcdv56Q= -github.com/pion/webrtc/v4 v4.0.10/go.mod h1:ViHLVaNpiuvaH8pdiuQxuA9awuE6KVzAXx3vVWilOck= +github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= +github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= +github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= +github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1213,6 +1200,8 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1221,14 +1210,14 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -1236,8 +1225,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1246,16 +1235,16 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag= github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= -github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= +github.com/quic-go/quic-go v0.52.0/go.mod h1:MFlGGpcpJqRAfmYi6NC2cptDPSxRWTOGNuP4wqrWmzQ= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= @@ -1498,8 +1487,8 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel/bridge/opencensus v1.28.0 h1:/BcyAV1bUJjSVxoeKwTQL9cS4X1iC6izZ9mheeuVSCU= go.opentelemetry.io/otel/bridge/opencensus v1.28.0/go.mod h1:FZp2xE+46yAyp3DfLFALze58nY0iIE8zs+mCgkPAzq0= go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= @@ -1510,14 +1499,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0u go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/exporters/prometheus v0.50.0 h1:2Ewsda6hejmbhGFyUvWZjUThC98Cf8Zy6g0zkIimOng= go.opentelemetry.io/otel/exporters/prometheus v0.50.0/go.mod h1:pMm5PkUo5YwbLiuEf7t2xg4wbP0/eSJrMxIMxKosynY= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1526,16 +1515,17 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= -go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= -go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= -go.uber.org/fx v1.23.0/go.mod h1:o/D9n+2mLP6v1EG+qsdT1O8wKopYAsqZasju97SDFCU= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= +go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= +go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -1546,7 +1536,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -1578,8 +1567,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1590,8 +1579,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4= -golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1617,8 +1606,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1674,8 +1663,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1700,8 +1689,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1768,7 +1757,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1783,12 +1771,11 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1798,8 +1785,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1812,14 +1799,14 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1873,19 +1860,18 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= -golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1982,8 +1968,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2011,7 +1997,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -2027,8 +2012,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -lukechampine.com/blake3 v1.4.0 h1:xDbKOZCVbnZsfzM6mHSYcGRHZ3YrLDzqz8XnV4uaD5w= -lukechampine.com/blake3 v1.4.0/go.mod h1:MQJNQCTnR+kwOP/JEZSxj3MaQjp80FOFSNMMHXcSeX0= +lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= +lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index 767caa80a..2a609332b 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -9,7 +9,7 @@ CREATE TABLE ipni ( order_number BIGSERIAL PRIMARY KEY, -- Unique increasing order number ad_cid TEXT NOT NULL, context_id BYTEA NOT NULL, -- abi.PieceInfo in Curio - -- metadata column in not required as Curio only supports one type of metadata(HTTP) + -- metadata BYTEA NOT NULL DEFAULT '\xa01200' (Added in 20250505-market_mk20.sql) is_rm BOOLEAN NOT NULL, -- skip added in 20241106-market-fixes.sql @@ -26,6 +26,8 @@ CREATE TABLE ipni ( piece_cid TEXT NOT NULL, -- For easy look up piece_size BIGINT NOT NULL, -- For easy look up + -- piece_cid_v2 TEXT (Added in 20250505-market_mk20.sql) -- For easy lookup + unique (ad_cid) ); @@ -56,7 +58,7 @@ CREATE TABLE ipni_head ( -- on-disk .car block headers or from data in the piece index database. CREATE TABLE ipni_chunks ( cid TEXT PRIMARY KEY, -- CID of the chunk - piece_cid TEXT NOT NULL, -- Related Piece CID + piece_cid TEXT NOT NULL, -- Related Piece CID V2 chunk_num INTEGER NOT NULL, -- Chunk number within the piece. Chunk 0 has no "next" link. first_cid TEXT, -- In case of db-based chunks, the CID of the first cid in the chunk start_offset BIGINT, -- In case of .car-based chunks, the offset in the .car file where the chunk starts @@ -177,24 +179,24 @@ BEGIN -- If a different is_rm exists for the same context_id and provider, insert the new task IF FOUND THEN - INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) - VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); - RETURN; + INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) + VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); + RETURN; END IF; - -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table + -- If no conflicting entry is found in ipni_task, check the latest ad in ipni table SELECT is_rm INTO _latest_is_rm FROM ipni WHERE provider = _provider AND context_id = _context_id ORDER BY order_number DESC - LIMIT 1; + LIMIT 1; -- If the latest ad has the same is_rm value, raise an exception IF FOUND AND _latest_is_rm = _is_rm THEN RAISE EXCEPTION 'already published'; END IF; - -- If all conditions are met, insert the new task into ipni_task + -- If all conditions are met, insert the new task into ipni_task INSERT INTO ipni_task (sp_id, sector, reg_seal_proof, sector_offset, provider, context_id, is_rm, created_at, task_id, complete) VALUES (_sp_id, _sector, _reg_seal_proof, _sector_offset, _provider, _context_id, _is_rm, TIMEZONE('UTC', NOW()), _task_id, FALSE); END; diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index e3a0a1292..15b22bb33 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -42,6 +42,14 @@ ALTER TABLE market_piece_deal ALTER TABLE parked_pieces ADD COLUMN skip BOOLEAN NOT NULL DEFAULT FALSE; +-- Add column piece_cid_v2 to IPNI table +ALTER TABLE ipni + ADD COLUMN piece_cid_v2 TEXT; + +-- Add metadata column to IPNI table which defaults to binary of IpfsGatewayHttp +ALTER TABLE ipni + ADD COLUMN metadata BYTEA NOT NULL DEFAULT '\xa01200'; + -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). CREATE OR REPLACE FUNCTION process_piece_deal( @@ -242,7 +250,8 @@ CREATE TABLE market_mk20_pipeline_waiting ( CREATE TABLE market_mk20_upload_waiting ( id TEXT PRIMARY KEY, chunked BOOLEAN DEFAULT NULL, - ref_id BIGINT DEFAULT NULL + ref_id BIGINT DEFAULT NULL, + ready_at TIMESTAMPTZ DEFAULT NULL ); -- This table help disconnected downloads from main PoRep/PDP pipelines @@ -277,6 +286,7 @@ CREATE TABLE market_mk20_deal_chunk ( chunk_size BIGINT not null, ref_id BIGINT DEFAULT NULL, complete BOOLEAN DEFAULT FALSE, + completed_at TIMESTAMPTZ, finalize BOOLEAN DEFAULT FALSE, finalize_task_id BIGINT DEFAULT NULL, PRIMARY KEY (id, chunk) @@ -303,6 +313,58 @@ INSERT INTO market_mk20_data_source (name, enabled) VALUES ('aggregate', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('offline', TRUE); INSERT INTO market_mk20_data_source (name, enabled) VALUES ('put', TRUE); +-- This function sets an upload completion time. It is used to removed +-- upload for deal which are not finalized in 1 hour so we don't waste space. +CREATE OR REPLACE FUNCTION set_ready_at_for_serial_upload() +RETURNS TRIGGER AS $$ +BEGIN + -- Transition into "serial ready" state: chunked=false AND ref_id IS NOT NULL + IF NEW.chunked IS FALSE + AND NEW.ref_id IS NOT NULL + AND OLD.ready_at IS NULL + AND NOT (OLD.chunked IS FALSE AND OLD.ref_id IS NOT NULL) THEN + NEW.ready_at := NOW() AT TIME ZONE 'UTC'; +END IF; + +RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER trg_ready_at_serial + BEFORE UPDATE OF ref_id, chunked ON market_mk20_upload_waiting + FOR EACH ROW + EXECUTE FUNCTION set_ready_at_for_serial_upload(); + +-- This function sets an upload completion time. It is used to removed +-- upload for deal which are not finalized in 1 hour so we don't waste space. +CREATE OR REPLACE FUNCTION set_ready_at_when_all_chunks_complete() +RETURNS TRIGGER AS $$ +BEGIN + -- Only react when a chunk transitions to complete = true + IF (TG_OP = 'UPDATE' OR TG_OP = 'INSERT') AND NEW.complete IS TRUE THEN + -- If no incomplete chunks remain, set ready_at once + IF NOT EXISTS ( + SELECT 1 FROM market_mk20_deal_chunk + WHERE id = NEW.id AND (complete IS NOT TRUE) + ) THEN +UPDATE market_mk20_upload_waiting +SET ready_at = NOW() AT TIME ZONE 'UTC' +WHERE id = NEW.id + AND chunked = true + AND ready_at IS NULL; +END IF; +END IF; + +RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE TRIGGER trg_ready_at_chunks_update + AFTER INSERT OR UPDATE OF complete ON market_mk20_deal_chunk + FOR EACH ROW + EXECUTE FUNCTION set_ready_at_when_all_chunks_complete(); + -- This function triggers a download for an offline piece. -- It is different from MK1.2 PoRep pipeline as it download the offline pieces -- locally. This is to allow serving retrievals with piece park. @@ -375,13 +437,13 @@ BEGIN END; $$ LANGUAGE plpgsql; --- Main ProofSet table for PDP -CREATE TABLE pdp_proof_set ( - id BIGINT PRIMARY KEY, -- on-chain proofset id - client TEXT NOT NULL, -- client wallet which requested this proofset +-- Main DataSet table for PDP +CREATE TABLE pdp_data_set ( + id BIGINT PRIMARY KEY, -- on-chain dataset id + client TEXT NOT NULL, -- client wallet which requested this dataset - -- updated when a challenge is requested (either by first proofset add or by invokes of nextProvingPeriod) - -- initially NULL on fresh proofsets. + -- updated when a challenge is requested (either by first dataset add or by invokes of nextProvingPeriod) + -- initially NULL on fresh dataset prev_challenge_request_epoch BIGINT, -- task invoking nextProvingPeriod, the task should be spawned any time prove_at_epoch+challenge_window is in the past @@ -415,8 +477,8 @@ CREATE TABLE pdp_proof_set ( unique (remove_deal_id) ); --- ProofSet create table governs the PoofSet create task -CREATE TABLE pdp_proof_set_create ( +-- DataSet create table governs the DataSet create task +CREATE TABLE pdp_data_set_create ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -427,8 +489,8 @@ CREATE TABLE pdp_proof_set_create ( tx_hash TEXT DEFAULT NULL ); --- ProofSet delete table governs the PoofSet delete task -CREATE TABLE pdp_proof_set_delete ( +-- DataSet delete table governs the DataSet delete task +CREATE TABLE pdp_data_set_delete ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, @@ -439,22 +501,22 @@ CREATE TABLE pdp_proof_set_delete ( tx_hash TEXT DEFAULT NULL ); --- This table governs the delete root tasks -CREATE TABLE pdp_root_delete ( +-- This table governs the delete piece tasks +CREATE TABLE pdp_piece_delete ( id TEXT PRIMARY KEY, -- This is Market V2 Deal ID for lookup and response client TEXT NOT NULL, set_id BIGINT NOT NULL, - roots BIGINT[] NOT NULL, + pieces BIGINT[] NOT NULL, extra_data BYTEA, task_id BIGINT DEFAULT NULL, tx_hash TEXT DEFAULT NULL ); --- Main ProofSet Root table. Any and all root ever added by SP must be part of this table -CREATE TABLE pdp_proofset_root ( - proof_set_id BIGINT NOT NULL, -- pdp_proof_sets.id +-- Main DataSet Piece table. Any and all pieces ever added by SP must be part of this table +CREATE TABLE pdp_dataset_piece ( + data_set_id BIGINT NOT NULL, -- pdp_data_sets.id client TEXT NOT NULL, piece_cid_v2 TEXT NOT NULL, -- root cid (piececid v2) @@ -462,20 +524,20 @@ CREATE TABLE pdp_proofset_root ( piece_size BIGINT NOT NULL, raw_size BIGINT NOT NULL, - root BIGINT DEFAULT NULL, -- on-chain index of the root in the rootCids sub-array + piece BIGINT DEFAULT NULL, -- on-chain index of the piece in the pieceCids sub-array piece_ref BIGINT NOT NULL, -- piece_ref_id - add_deal_id TEXT NOT NULL, -- mk20 deal ID for adding this root to proofset + add_deal_id TEXT NOT NULL, -- mk20 deal ID for adding this root to dataset add_message_hash TEXT NOT NULL, add_message_index BIGINT NOT NULL, -- index of root in the add message removed BOOLEAN DEFAULT FALSE, - remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this root from proofset + remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this root from dataset remove_message_hash TEXT DEFAULT NULL, remove_message_index BIGINT DEFAULT NULL, - PRIMARY KEY (proof_set_id, root) + PRIMARY KEY (data_set_id, piece) ); CREATE TABLE pdp_pipeline ( @@ -489,7 +551,7 @@ CREATE TABLE pdp_pipeline ( piece_size BIGINT NOT NULL, raw_size BIGINT NOT NULL, - proof_set_id BIGINT NOT NULL, + data_set_id BIGINT NOT NULL, extra_data BYTEA, @@ -502,13 +564,13 @@ CREATE TABLE pdp_pipeline ( agg_task_id BIGINT DEFAULT NULL, aggregated BOOLEAN DEFAULT FALSE, - add_root_task_id BIGINT DEFAULT NULL, - after_add_root BOOLEAN DEFAULT FALSE, + add_piece_task_id BIGINT DEFAULT NULL, + after_add_piece BOOLEAN DEFAULT FALSE, add_message_hash TEXT, add_message_index BIGINT NOT NULL DEFAULT 0, -- index of root in the add message - after_add_root_msg BOOLEAN DEFAULT FALSE, + after_add_piece_msg BOOLEAN DEFAULT FALSE, save_cache_task_id BIGINT DEFAULT NULL, after_save_cache BOOLEAN DEFAULT FALSE, @@ -519,6 +581,10 @@ CREATE TABLE pdp_pipeline ( indexed BOOLEAN DEFAULT FALSE, announce BOOLEAN DEFAULT FALSE, + announce_payload BOOLEAN DEFAULT FALSE, + + announced BOOLEAN DEFAULT FALSE, + announced_payload BOOLEAN DEFAULT FALSE, complete BOOLEAN DEFAULT FALSE, @@ -530,6 +596,15 @@ CREATE TABLE market_mk20_clients ( allowed BOOLEAN DEFAULT TRUE ); +CREATE TABLE pdp_proving_tasks ( + data_set_id BIGINT NOT NULL, -- pdp_data_set.id + task_id BIGINT NOT NULL, -- harmony_task task ID + + PRIMARY KEY (data_set_id, task_id), + FOREIGN KEY (data_set_id) REFERENCES pdp_data_set(id) ON DELETE CASCADE, + FOREIGN KEY (task_id) REFERENCES harmony_task(id) ON DELETE CASCADE +); + -- IPNI pipeline is kept separate from rest for robustness -- and reuse. This allows for removing, recreating ads using CLI. CREATE TABLE pdp_ipni_task ( @@ -547,7 +622,6 @@ CREATE TABLE pdp_ipni_task ( PRIMARY KEY (context_id, is_rm) ); - -- Function to create ipni tasks CREATE OR REPLACE FUNCTION insert_pdp_ipni_task( _context_id BYTEA, @@ -562,7 +636,7 @@ _latest_is_rm BOOLEAN; BEGIN -- Check if ipni_task has the same context_id and provider with a different is_rm value SELECT is_rm INTO _existing_is_rm - FROM ipni_task + FROM pdp_ipni_task WHERE provider = _provider AND context_id = _context_id AND is_rm != _is_rm LIMIT 1; @@ -591,5 +665,36 @@ BEGIN END; $$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION insert_ad_and_update_head( + _ad_cid TEXT, + _context_id BYTEA, + _metadata BYTEA, + _piece_cid_v2 TEXT, + _piece_cid TEXT, + _piece_size BIGINT, + _is_rm BOOLEAN, + _provider TEXT, + _addresses TEXT, + _signature BYTEA, + _entries TEXT +) RETURNS VOID AS $$ +DECLARE +_previous TEXT; +BEGIN + -- Determine the previous ad_cid in the chain for this provider + SELECT head INTO _previous + FROM ipni_head + WHERE provider = _provider; + + -- Insert the new ad into the ipni table with an automatically assigned order_number + INSERT INTO ipni (ad_cid, context_id, metadata, is_rm, previous, provider, addresses, signature, entries, piece_cid_v2, piece_cid, piece_size) + VALUES (_ad_cid, _context_id, metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size); + -- Update the ipni_head table to set the new ad as the head of the chain + INSERT INTO ipni_head (provider, head) + VALUES (_provider, _ad_cid) + ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head; + +END; +$$ LANGUAGE plpgsql; diff --git a/market/ipni/chunker/serve-chunker.go b/market/ipni/chunker/serve-chunker.go index 9fcaf5f7b..4e13a1969 100644 --- a/market/ipni/chunker/serve-chunker.go +++ b/market/ipni/chunker/serve-chunker.go @@ -298,9 +298,16 @@ func (p *ServeChunker) reconstructChunkFromDB(ctx context.Context, chunk, piecev pi := commp.PieceInfo() - mhs, err := p.indexStore.GetPieceHashRange(ctx, piecev2, firstHash, numBlocks) - if err != nil { - return nil, xerrors.Errorf("getting piece hash range: %w", err) + var mhs []multihash.Multihash + + // Handle exception for PDP piece announcement with FilecoinPieceHttp{} metadata + if numBlocks == 1 { + mhs = []multihash.Multihash{firstHash} + } else { + mhs, err = p.indexStore.GetPieceHashRange(ctx, piecev2, firstHash, numBlocks) + if err != nil { + return nil, xerrors.Errorf("getting piece hash range: %w", err) + } } // Create the chunk node diff --git a/market/ipni/types/types.go b/market/ipni/types/types.go new file mode 100644 index 000000000..cc28d7e03 --- /dev/null +++ b/market/ipni/types/types.go @@ -0,0 +1,16 @@ +package types + +import ( + "github.com/ipfs/go-cid" +) + +//go:generate cbor-gen-for --map-encoding PieceInfo + +// PieceInfo is used to generate the context CIDs for PDP IPNI ads +type PieceInfo struct { + // PieceCID is piece CID V2 + PieceCID cid.Cid + + // Payload determines if the IPNI ad is TransportFilecoinPieceHttp or TransportIpfsGatewayHttp + Payload bool +} diff --git a/market/ipni/types/types_cbor_gen.go b/market/ipni/types/types_cbor_gen.go new file mode 100644 index 000000000..bb408fe15 --- /dev/null +++ b/market/ipni/types/types_cbor_gen.go @@ -0,0 +1,150 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package types + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *PieceInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Payload (bool) (bool) + if len("Payload") > 8192 { + return xerrors.Errorf("Value in field \"Payload\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Payload"))); err != nil { + return err + } + if _, err := cw.WriteString(string("Payload")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Payload); err != nil { + return err + } + + // t.PieceCID (cid.Cid) (struct) + if len("PieceCID") > 8192 { + return xerrors.Errorf("Value in field \"PieceCID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PieceCID"))); err != nil { + return err + } + if _, err := cw.WriteString(string("PieceCID")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.PieceCID); err != nil { + return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) + } + + return nil +} + +func (t *PieceInfo) UnmarshalCBOR(r io.Reader) (err error) { + *t = PieceInfo{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PieceInfo: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Payload (bool) (bool) + case "Payload": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Payload = false + case 21: + t.Payload = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.PieceCID (cid.Cid) (struct) + case "PieceCID": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) + } + + t.PieceCID = c + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/market/mk20/client/client.go b/market/mk20/client/client.go index 975c22ad3..75521d20b 100644 --- a/market/mk20/client/client.go +++ b/market/mk20/client/client.go @@ -168,9 +168,9 @@ func (c *Client) Deal(ctx context.Context, maddr, wallet address.Address, pieceC if pdp { ps := uint64(proofSet) p.PDPV1 = &mk20.PDPV1{ - AddRoot: true, - ProofSetID: &ps, - ExtraData: []byte("test bytes"), // TODO: Fix this + AddPiece: true, + DataSetID: &ps, + ExtraData: []byte("test bytes"), // TODO: Fix this } } diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 340aaeeec..90f924f25 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -1138,20 +1138,24 @@ const docTemplate = `{ "mk20.PDPV1": { "type": "object", "properties": { - "add_root": { - "description": "AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined.", + "add_piece": { + "description": "AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined.", "type": "boolean" }, - "create_proof_set": { - "description": "CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider.", + "create_data_set": { + "description": "CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider.", "type": "boolean" }, - "delete_proof_set": { - "description": "DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client.\nProofSetID must be defined.", + "data_set_id": { + "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", + "type": "integer" + }, + "delete_data_set": { + "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", "type": "boolean" }, - "delete_root": { - "description": "DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined.", + "delete_piece": { + "description": "DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined.", "type": "boolean" }, "extra_data": { @@ -1161,20 +1165,16 @@ const docTemplate = `{ "type": "integer" } }, - "proof_set_id": { - "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", - "type": "integer" - }, - "record_keeper": { - "description": "RecordKeeper specifies the record keeper contract address for the new PDP proofset.", - "type": "string" - }, - "root_ids": { - "description": "RootIDs is a list of root ids in a proof set.", + "piece_ids": { + "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { "type": "integer" } + }, + "record_keeper": { + "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", + "type": "string" } } }, diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index d14c9d2bc..a9fc597a8 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -1129,20 +1129,24 @@ "mk20.PDPV1": { "type": "object", "properties": { - "add_root": { - "description": "AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined.", + "add_piece": { + "description": "AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined.", "type": "boolean" }, - "create_proof_set": { - "description": "CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider.", + "create_data_set": { + "description": "CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider.", "type": "boolean" }, - "delete_proof_set": { - "description": "DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client.\nProofSetID must be defined.", + "data_set_id": { + "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", + "type": "integer" + }, + "delete_data_set": { + "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", "type": "boolean" }, - "delete_root": { - "description": "DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined.", + "delete_piece": { + "description": "DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined.", "type": "boolean" }, "extra_data": { @@ -1152,20 +1156,16 @@ "type": "integer" } }, - "proof_set_id": { - "description": "ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true.", - "type": "integer" - }, - "record_keeper": { - "description": "RecordKeeper specifies the record keeper contract address for the new PDP proofset.", - "type": "string" - }, - "root_ids": { - "description": "RootIDs is a list of root ids in a proof set.", + "piece_ids": { + "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { "type": "integer" } + }, + "record_keeper": { + "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", + "type": "string" } } }, diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index 2d4e41728..6bc350561 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -257,22 +257,26 @@ definitions: type: object mk20.PDPV1: properties: - add_root: - description: AddRoot indicated that this deal is meant to add root to a given - ProofSet. ProofSetID must be defined. + add_piece: + description: AddPiece indicated that this deal is meant to add Piece to a + given DataSet. DataSetID must be defined. type: boolean - create_proof_set: - description: CreateProofSet indicated that this deal is meant to create a - new ProofSet for the client by storage provider. + create_data_set: + description: CreateDataSet indicated that this deal is meant to create a new + DataSet for the client by storage provider. type: boolean - delete_proof_set: + data_set_id: + description: DataSetID is PDP verified contract dataset ID. It must be defined + for all deals except when CreateDataSet is true. + type: integer + delete_data_set: description: |- - DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. - ProofSetID must be defined. + DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client. + DataSetID must be defined. type: boolean - delete_root: - description: DeleteRoot indicates whether the root of the data should be deleted. - ProofSetID must be defined. + delete_piece: + description: DeletePiece indicates whether the Piece of the data should be + deleted. DataSetID must be defined. type: boolean extra_data: description: ExtraData can be used to send additional information to service @@ -280,19 +284,15 @@ definitions: items: type: integer type: array - proof_set_id: - description: ProofSetID is PDP verified contract proofset ID. It must be defined - for all deals except when CreateProofSet is true. - type: integer - record_keeper: - description: RecordKeeper specifies the record keeper contract address for - the new PDP proofset. - type: string - root_ids: - description: RootIDs is a list of root ids in a proof set. + piece_ids: + description: PieceIDs is a list of Piece ids in a proof set. items: type: integer type: array + record_keeper: + description: RecordKeeper specifies the record keeper contract address for + the new PDP dataset. + type: string type: object mk20.PieceDataFormat: properties: diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 625f273d5..ac1b4d80d 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -82,6 +82,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI } go markDownloaded(ctx, db) + go removeNotFinalizedUploads(ctx, db) return &MK20{ miners: miners, @@ -417,31 +418,34 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje return false, xerrors.Errorf("saving deal to DB: %w", err) } - // If we have data source other that PUT then start the pipeline - if deal.Data != nil { - if deal.Data.SourceHTTP != nil || deal.Data.SourceAggregate != nil { - err = insertPDPPipeline(ctx, tx, deal) - if err != nil { - return false, xerrors.Errorf("inserting pipeline: %w", err) + pdp := deal.Products.PDPV1 + + if pdp.AddPiece { + // If we have data source other that PUT then start the pipeline + if deal.Data != nil { + if deal.Data.SourceHTTP != nil || deal.Data.SourceAggregate != nil { + err = insertPDPPipeline(ctx, tx, deal) + if err != nil { + return false, xerrors.Errorf("inserting pipeline: %w", err) + } } - } - if deal.Data.SourceHttpPut != nil { + if deal.Data.SourceHttpPut != nil { + _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) + if err != nil { + return false, xerrors.Errorf("inserting upload waiting: %w", err) + } + } + } else { + // Assume upload _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) if err != nil { return false, xerrors.Errorf("inserting upload waiting: %w", err) } } - } else { - // Assume upload - _, err = tx.Exec(`INSERT INTO market_mk20_upload_waiting (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, deal.Identifier.String()) - if err != nil { - return false, xerrors.Errorf("inserting upload waiting: %w", err) - } } - pdp := deal.Products.PDPV1 - if pdp.CreateProofSet { - n, err := m.DB.Exec(ctx, `INSERT INTO pdp_proof_set_create (id, client, record_keeper, extra_data) VALUES ($1, $2, $3, $4)`, + if pdp.CreateDataSet { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_data_set_create (id, client, record_keeper, extra_data) VALUES ($1, $2, $3, $4)`, deal.Identifier.String(), deal.Client.String(), pdp.RecordKeeper, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP proof set create: %w", err) @@ -451,9 +455,9 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } } - if pdp.DeleteProofSet { - n, err := m.DB.Exec(ctx, `INSERT INTO pdp_proof_set_delete (id, client, set_id, extra_data) VALUES ($1, $2, $3, $4)`, - deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.ExtraData) + if pdp.DeleteDataSet { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_data_set_delete (id, client, set_id, extra_data) VALUES ($1, $2, $3, $4)`, + deal.Identifier.String(), deal.Client.String(), *pdp.DataSetID, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP proof set delete: %w", err) } @@ -462,9 +466,9 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } } - if pdp.DeleteRoot { - n, err := m.DB.Exec(ctx, `INSERT INTO pdp_root_delete (id, client, set_id, roots, extra_data) VALUES ($1, $2, $3, $4, $5)`, - deal.Identifier.String(), deal.Client.String(), *pdp.ProofSetID, pdp.RootIDs, pdp.ExtraData) + if pdp.DeletePiece { + n, err := m.DB.Exec(ctx, `INSERT INTO pdp_piece_delete (id, client, set_id, pieces, extra_data) VALUES ($1, $2, $3, $4, $5)`, + deal.Identifier.String(), deal.Client.String(), *pdp.DataSetID, pdp.PieceIDs, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP delete root: %w", err) } @@ -494,7 +498,7 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje } func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { - if deal.Products.PDPV1.AddRoot && deal.Products.RetrievalV1 == nil { + if deal.Products.PDPV1.AddPiece && deal.Products.RetrievalV1 == nil { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, Reason: "Retrieval deal is required for pdp_v1", @@ -519,13 +523,13 @@ func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRe p := deal.Products.PDPV1 - // This serves as Auth for now. We are checking if client is authorized to make changes to the proof set or roots + // This serves as Auth for now. We are checking if client is authorized to make changes to the proof set or pieces // In future this will be replaced by an ACL check - if p.DeleteProofSet || p.AddRoot { - pid := *p.ProofSetID + if p.DeleteDataSet || p.AddPiece { + pid := *p.DataSetID var exists bool - err := m.DB.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE AND client = $2)`, pid, deal.Client.String()).Scan(&exists) + err := m.DB.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_data_set WHERE id = $1 AND removed = FALSE AND client = $2)`, pid, deal.Client.String()).Scan(&exists) if err != nil { log.Errorw("error checking if proofset exists", "error", err) return &ProviderDealRejectionInfo{ @@ -541,20 +545,20 @@ func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRe } } - if p.DeleteRoot { - pid := *p.ProofSetID + if p.DeletePiece { + pid := *p.DataSetID var exists bool err := m.DB.QueryRow(ctx, `SELECT COUNT(*) = cardinality($2::BIGINT[]) AS all_exist_and_active - FROM pdp_proofset_root r - JOIN pdp_proof_set s ON r.proof_set_id = s.id - WHERE r.proof_set_id = $1 - AND r.root = ANY($2) + FROM pdp_dataset_piece r + JOIN pdp_data_set s ON r.data_set_id = s.id + WHERE r.data_set_id = $1 + AND r.piece = ANY($2) AND r.removed = FALSE AND s.removed = FALSE AND r.client = $3 - AND s.client = $3;`, pid, p.RootIDs, deal.Client.String()).Scan(&exists) + AND s.client = $3;`, pid, p.PieceIDs, deal.Client.String()).Scan(&exists) if err != nil { - log.Errorw("error checking if proofset and roots exist for the client", "error", err) + log.Errorw("error checking if dataset and pieces exist for the client", "error", err) return &ProviderDealRejectionInfo{ HTTPCode: ErrServerInternalError, Reason: "", @@ -564,7 +568,7 @@ func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRe if !exists { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, - Reason: "proofset or one of the roots does not exist for the client", + Reason: "dataset or one of the pieces does not exist for the client", }, nil } } @@ -639,11 +643,11 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error } n, err = tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, deal_aggregation, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - dealID, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, aggregation, retv.Indexing, retv.AnnouncePayload) + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, deal_aggregation, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, + dealID, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, aggregation, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return xerrors.Errorf("inserting PDP pipeline: %w", err) } @@ -743,10 +747,10 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error } pBatch.Queue(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, - proof_set_id, extra_data, piece_ref, deal_aggregation, aggr_index, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`, + data_set_id, extra_data, piece_ref, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, dealID, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), spi.Size, spi.RawSize, - pdp.ExtraData, *pdp.ProofSetID, aggregation, i, retv.Indexing, retv.AnnouncePayload) + pdp.ExtraData, *pdp.DataSetID, aggregation, i, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if pBatch.Len() > pBatchSize { res := tx.SendBatch(ctx, pBatch) if err := res.Close(); err != nil { diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index 7e5e739aa..96c2f26a2 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -395,7 +395,8 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } n, err = tx.Exec(`UPDATE market_mk20_deal_chunk SET - complete = TRUE, + complete = TRUE, + completed_at = NOW() AT TIME ZONE 'UTC', ref_id = $1 WHERE id = $2 AND chunk = $3 @@ -659,6 +660,15 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW return false, xerrors.Errorf("inserting parked piece ref: %w", err) } + // Mark upload as started to prevent someone else from using chunk upload + n, err := tx.Exec(`UPDATE market_mk20_upload_waiting SET chunked = FALSE WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("updating upload waiting: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating upload waiting: expected 1 row updated, got %d", n) + } + return true, nil }) @@ -683,6 +693,11 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW if err != nil { log.Errorw("failed to delete parked piece ref", "deal", id, "error", err) } + + _, err = m.DB.Exec(ctx, `UPDATE market_mk20_upload_waiting SET chunked = NULL WHERE id = $1`, id.String()) + if err != nil { + log.Errorw("failed to update upload waiting", "deal", id, "error", err) + } } }() @@ -713,7 +728,7 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW // If piece does not exist then we update piece park table to work with new tmpID // Update ref table's reference to tmp id - _, err = tx.Exec(`UPDATE parked_piece_refs SET piece_id = $1 WHERE piece_id = $2`, pid, pnum) + _, err = tx.Exec(`UPDATE parked_piece_refs SET piece_id = $1 WHERE piece_id = $2`, pnum, pid) if err != nil { return false, xerrors.Errorf("updating parked piece ref: %w", err) } @@ -921,11 +936,11 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo // Insert the PDP pipeline n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } @@ -949,3 +964,108 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo w.WriteHeader(int(Ok)) } + +func removeNotFinalizedUploads(ctx context.Context, db *harmonydb.DB) { + rm := func(ctx context.Context, db *harmonydb.DB) { + var deals []struct { + ID string `db:"id"` + Chunked bool `db:"chunked"` + RefID sql.NullInt64 `db:"ref_id"` + ReadyAt time.Time `db:"ready_at"` + } + + err := db.Select(ctx, &deals, `SELECT id, chunked, ref_id, ready_at + FROM market_mk20_upload_waiting + WHERE chunked IS NOT NULL + AND ready_at <= (NOW() AT TIME ZONE 'UTC') - INTERVAL '60 minutes';`) + if err != nil { + log.Errorw("failed to get not finalized uploads", "error", err) + } + + for _, deal := range deals { + if deal.Chunked { + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + _, err = tx.Exec(`DELETE FROM parked_piece_refs p + USING ( + SELECT DISTINCT ref_id + FROM market_mk20_deal_chunk + WHERE id = $1 AND ref_id IS NOT NULL + ) c + WHERE p.ref_id = c.ref_id; + `, deal.ID) + if err != nil { + return false, xerrors.Errorf("deleting piece refs: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, deal.ID) + if err != nil { + return false, xerrors.Errorf("deleting deal chunks: %w", err) + } + + n, err := tx.Exec(`UPDATE market_mk20_upload_waiting + SET chunked = NULL, + ref_id = NULL, + ready_at = NULL + WHERE id = $1;`, deal.ID) + + if err != nil { + return false, xerrors.Errorf("updating upload waiting: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating upload waiting: expected 1 row updated, got %d", n) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorw("failed to delete not finalized uploads", "deal", deal.ID, "error", err) + } + if !comm { + log.Errorw("failed to delete not finalized uploads", "deal", deal.ID, "error", "failed to commit transaction") + } + } else { + if deal.RefID.Valid { + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, deal.RefID.Int64) + if err != nil { + return false, xerrors.Errorf("deleting piece refs: %w", err) + } + + n, err := tx.Exec(`UPDATE market_mk20_upload_waiting + SET chunked = NULL, + ref_id = NULL, + ready_at = NULL + WHERE id = $1;`, deal.ID) + + if err != nil { + return false, xerrors.Errorf("updating upload waiting: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating upload waiting: expected 1 row updated, got %d", n) + } + + return true, nil + }) + if err != nil { + log.Errorw("failed to delete not finalized uploads", "deal", deal.ID, "error", err) + } + if !comm { + log.Errorw("failed to delete not finalized uploads", "deal", deal.ID, "error", "failed to commit transaction") + } + } + log.Errorw("removing not finalized upload", "deal", deal.ID, "error", "ref_id not found") + } + } + } + + ticker := time.NewTicker(time.Minute * 5) + defer ticker.Stop() + for { + select { + case <-ticker.C: + rm(ctx, db) + case <-ctx.Done(): + return + } + } +} diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 6ee90a978..8a6666e7c 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -12,27 +12,27 @@ import ( // PDPV1 represents configuration for product-specific PDP version 1 deals. type PDPV1 struct { - // CreateProofSet indicated that this deal is meant to create a new ProofSet for the client by storage provider. - CreateProofSet bool `json:"create_proof_set"` + // CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider. + CreateDataSet bool `json:"create_data_set"` - // DeleteProofSet indicated that this deal is meant to delete an existing ProofSet created by SP for the client. - // ProofSetID must be defined. - DeleteProofSet bool `json:"delete_proof_set"` + // DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client. + // DataSetID must be defined. + DeleteDataSet bool `json:"delete_data_set"` - // AddRoot indicated that this deal is meant to add root to a given ProofSet. ProofSetID must be defined. - AddRoot bool `json:"add_root"` + // AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined. + AddPiece bool `json:"add_piece"` - // DeleteRoot indicates whether the root of the data should be deleted. ProofSetID must be defined. - DeleteRoot bool `json:"delete_root"` + // DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined. + DeletePiece bool `json:"delete_piece"` - // ProofSetID is PDP verified contract proofset ID. It must be defined for all deals except when CreateProofSet is true. - ProofSetID *uint64 `json:"proof_set_id,omitempty"` + // DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. + DataSetID *uint64 `json:"data_set_id,omitempty"` - // RecordKeeper specifies the record keeper contract address for the new PDP proofset. + // RecordKeeper specifies the record keeper contract address for the new PDP dataset. RecordKeeper string `json:"record_keeper"` - // RootIDs is a list of root ids in a proof set. - RootIDs []uint64 `json:"root_ids,omitempty"` + // PieceIDs is a list of Piece ids in a proof set. + PieceIDs []uint64 `json:"piece_ids,omitempty"` // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. ExtraData []byte `json:"extra_data,omitempty"` @@ -44,13 +44,24 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er return code, err } - if ok := p.CreateProofSet || p.DeleteProofSet || p.AddRoot || p.DeleteRoot; !ok { + if ok := p.CreateDataSet || p.DeleteDataSet || p.AddPiece || p.DeletePiece; !ok { return ErrBadProposal, xerrors.Errorf("deal must have one of the following flags set: create_proof_set, delete_proof_set, add_root, delete_root") } - if p.CreateProofSet { - if p.ProofSetID != nil { - return ErrBadProposal, xerrors.Errorf("create_proof_set cannot be set with proof_set_id") + var existingAddress bool + + err = db.QueryRow(context.Background(), `SELECT EXISTS(SELECT 1 FROM eth_keys WHERE role = 'pdp')`).Scan(&existingAddress) + if err != nil { + return ErrServerInternalError, xerrors.Errorf("checking if pdp address exists: %w", err) + } + + if !existingAddress { + return ErrServiceMaintenance, xerrors.Errorf("pdp key not configured by storage provider") + } + + if p.CreateDataSet { + if p.DataSetID != nil { + return ErrBadProposal, xerrors.Errorf("create_proof_set cannot be set with data_set_id") } if p.RecordKeeper == "" { return ErrBadProposal, xerrors.Errorf("record_keeper must be defined for create_proof_set") @@ -61,63 +72,63 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er } // Only 1 action is allowed per deal - if btoi(p.CreateProofSet)+btoi(p.DeleteProofSet)+btoi(p.AddRoot)+btoi(p.DeleteRoot) > 1 { + if btoi(p.CreateDataSet)+btoi(p.DeleteDataSet)+btoi(p.AddPiece)+btoi(p.DeletePiece) > 1 { return ErrBadProposal, xerrors.Errorf("only one action is allowed per deal") } ctx := context.Background() - if p.DeleteProofSet { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_proof_set must have proof_set_id defined") + if p.DeleteDataSet { + if p.DataSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_proof_set must have data_set_id defined") } - pid := *p.ProofSetID + pid := *p.DataSetID var exists bool - err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_data_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) if err != nil { - return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if dataset exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset does not exist for the client") + return ErrBadProposal, xerrors.Errorf("dataset does not exist for the client") } } - if p.AddRoot { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("add_root must have proof_set_id defined") + if p.AddPiece { + if p.DataSetID == nil { + return ErrBadProposal, xerrors.Errorf("add_root must have data_set_id defined") } - pid := *p.ProofSetID + pid := *p.DataSetID var exists bool - err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_proof_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) + err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_data_set WHERE id = $1 AND removed = FALSE)`, pid).Scan(&exists) if err != nil { - return ErrServerInternalError, xerrors.Errorf("checking if proofset exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if dataset exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset does not exist for the client") + return ErrBadProposal, xerrors.Errorf("dataset does not exist for the client") } } - if p.DeleteRoot { - if p.ProofSetID == nil { - return ErrBadProposal, xerrors.Errorf("delete_root must have proof_set_id defined") + if p.DeletePiece { + if p.DataSetID == nil { + return ErrBadProposal, xerrors.Errorf("delete_root must have data_set_id defined") } - pid := *p.ProofSetID - if len(p.RootIDs) == 0 { + pid := *p.DataSetID + if len(p.PieceIDs) == 0 { return ErrBadProposal, xerrors.Errorf("root_ids must be defined for delete_proof_set") } var exists bool err := db.QueryRow(ctx, `SELECT COUNT(*) = cardinality($2::BIGINT[]) AS all_exist_and_active - FROM pdp_proofset_root r - JOIN pdp_proof_set s ON r.proof_set_id = s.id - WHERE r.proof_set_id = $1 + FROM pdp_dataset_piece r + JOIN pdp_data_set s ON r.data_set_id = s.id + WHERE r.data_set_id = $1 AND r.root = ANY($2) AND r.removed = FALSE - AND s.removed = FALSE;`, pid, p.RootIDs).Scan(&exists) + AND s.removed = FALSE;`, pid, p.PieceIDs).Scan(&exists) if err != nil { - return ErrServerInternalError, xerrors.Errorf("checking if proofset and roots exists: %w", err) + return ErrServerInternalError, xerrors.Errorf("checking if dataset and pieces exists: %w", err) } if !exists { - return ErrBadProposal, xerrors.Errorf("proofset or one of the roots does not exist for the client") + return ErrBadProposal, xerrors.Errorf("dataset or one of the pieces does not exist for the client") } } diff --git a/market/mk20/retrieval_v1.go b/market/mk20/retrieval_v1.go index bbd316962..5fa69ac9f 100644 --- a/market/mk20/retrieval_v1.go +++ b/market/mk20/retrieval_v1.go @@ -29,9 +29,6 @@ func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCo return ErrProductValidationFailed, xerrors.Errorf("deal cannot be announced to IPNI without indexing") } - if r.AnnouncePiece && r.AnnouncePayload { - return ErrProductValidationFailed, xerrors.Errorf("cannot announce both payload and piece to IPNI at the same time") - } return Ok, nil } diff --git a/pdp/contract/IPDPProvingSchedule.json b/pdp/contract/IPDPProvingSchedule.json index b1fbe4526..ebc58af53 100644 --- a/pdp/contract/IPDPProvingSchedule.json +++ b/pdp/contract/IPDPProvingSchedule.json @@ -1 +1 @@ -{"abi":[{"type":"function","name":"challengeWindow","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"getChallengesPerProof","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"getMaxProvingPeriod","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"initChallengeWindowStart","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"nextChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"challengeWindow()":"861a1412","getChallengesPerProof()":"47d3dfe7","getMaxProvingPeriod()":"f2f12333","initChallengeWindowStart()":"21918cea","nextChallengeWindowStart(uint256)":"8bf96d28"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"challengeWindow\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengesPerProof\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getMaxProvingPeriod\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"initChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"challengeWindow()\":{\"returns\":{\"_0\":\"Challenge window size in epochs\"}},\"getChallengesPerProof()\":{\"returns\":{\"_0\":\"Number of challenges required per proof\"}},\"getMaxProvingPeriod()\":{\"returns\":{\"_0\":\"Maximum proving period in epochs\"}},\"nextChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the proof set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingWindow\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"challengeWindow()\":{\"notice\":\"Returns the number of epochs at the end of a proving period during which proofs can be submitted\"},\"getChallengesPerProof()\":{\"notice\":\"Returns the required number of challenges/merkle inclusion proofs per proof set\"},\"getMaxProvingPeriod()\":{\"notice\":\"Returns the number of epochs allowed before challenges must be resampled\"},\"initChallengeWindowStart()\":{\"notice\":\"Value for initializing the challenge window start for any proof set assuming proving period starts now\"},\"nextChallengeWindowStart(uint256)\":{\"notice\":\"Calculates the start of the next challenge window for a given proof set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":200},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=node_modules/@pythnetwork/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\"]},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x6fc7848345c358a7a18e43ad9d93c1ea5fecf9d3f0daca721576d6de96d797b2\",\"license\":\"UNLICENSED\",\"urls\":[\"bzz-raw://ab29f0b39894650cf74b6a771e50bc50c91d54f6ba6e5a1b11c7cb1d7878d0cf\",\"dweb:/ipfs/QmawGZjCfua9dbJsqCzN6J9v3kLsE4oRLwMhbbcE4RYUNh\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"pure","type":"function","name":"challengeWindow","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getChallengesPerProof","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getMaxProvingPeriod","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"initChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"challengeWindow()":{"returns":{"_0":"Challenge window size in epochs"}},"getChallengesPerProof()":{"returns":{"_0":"Number of challenges required per proof"}},"getMaxProvingPeriod()":{"returns":{"_0":"Maximum proving period in epochs"}},"nextChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the proof set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"challengeWindow()":{"notice":"Returns the number of epochs at the end of a proving period during which proofs can be submitted"},"getChallengesPerProof()":{"notice":"Returns the required number of challenges/merkle inclusion proofs per proof set"},"getMaxProvingPeriod()":{"notice":"Returns the number of epochs allowed before challenges must be resampled"},"initChallengeWindowStart()":{"notice":"Value for initializing the challenge window start for any proof set assuming proving period starts now"},"nextChallengeWindowStart(uint256)":{"notice":"Calculates the start of the next challenge window for a given proof set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=node_modules/@pythnetwork/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/"],"optimizer":{"enabled":true,"runs":200},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{}},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x6fc7848345c358a7a18e43ad9d93c1ea5fecf9d3f0daca721576d6de96d797b2","urls":["bzz-raw://ab29f0b39894650cf74b6a771e50bc50c91d54f6ba6e5a1b11c7cb1d7878d0cf","dweb:/ipfs/QmawGZjCfua9dbJsqCzN6J9v3kLsE4oRLwMhbbcE4RYUNh"],"license":"UNLICENSED"}},"version":1},"id":0} \ No newline at end of file +{"abi":[{"type":"function","name":"challengeWindow","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"getChallengesPerProof","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"getMaxProvingPeriod","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"initChallengeWindowStart","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"nextChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"challengeWindow()":"861a1412","getChallengesPerProof()":"47d3dfe7","getMaxProvingPeriod()":"f2f12333","initChallengeWindowStart()":"21918cea","nextChallengeWindowStart(uint256)":"8bf96d28"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"challengeWindow\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengesPerProof\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getMaxProvingPeriod\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"initChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"challengeWindow()\":{\"returns\":{\"_0\":\"Challenge window size in epochs\"}},\"getChallengesPerProof()\":{\"returns\":{\"_0\":\"Number of challenges required per proof\"}},\"getMaxProvingPeriod()\":{\"returns\":{\"_0\":\"Maximum proving period in epochs\"}},\"nextChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the data set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingWindow\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"challengeWindow()\":{\"notice\":\"Returns the number of epochs at the end of a proving period during which proofs can be submitted\"},\"getChallengesPerProof()\":{\"notice\":\"Returns the required number of challenges/merkle inclusion proofs per data set\"},\"getMaxProvingPeriod()\":{\"notice\":\"Returns the number of epochs allowed before challenges must be resampled\"},\"initChallengeWindowStart()\":{\"notice\":\"Value for initializing the challenge window start for any data set assuming proving period starts now\"},\"nextChallengeWindowStart(uint256)\":{\"notice\":\"Calculates the start of the next challenge window for a given data set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x404a211500ef49c7fddaccc98267dbeb18b5c5077ef57d3025337dbf319faa84\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://62053e73723267e564db5a1e7fa0c9b21fb6241c823061c9d414a26fea600af7\",\"dweb:/ipfs/QmWKrJaFBuL5W521YTaLAZoJFqBskAUQi9y4MijxX92G9F\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"pure","type":"function","name":"challengeWindow","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getChallengesPerProof","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getMaxProvingPeriod","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"initChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"challengeWindow()":{"returns":{"_0":"Challenge window size in epochs"}},"getChallengesPerProof()":{"returns":{"_0":"Number of challenges required per proof"}},"getMaxProvingPeriod()":{"returns":{"_0":"Maximum proving period in epochs"}},"nextChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the data set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"challengeWindow()":{"notice":"Returns the number of epochs at the end of a proving period during which proofs can be submitted"},"getChallengesPerProof()":{"notice":"Returns the required number of challenges/merkle inclusion proofs per data set"},"getMaxProvingPeriod()":{"notice":"Returns the number of epochs allowed before challenges must be resampled"},"initChallengeWindowStart()":{"notice":"Value for initializing the challenge window start for any data set assuming proving period starts now"},"nextChallengeWindowStart(uint256)":{"notice":"Calculates the start of the next challenge window for a given data set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x404a211500ef49c7fddaccc98267dbeb18b5c5077ef57d3025337dbf319faa84","urls":["bzz-raw://62053e73723267e564db5a1e7fa0c9b21fb6241c823061c9d414a26fea600af7","dweb:/ipfs/QmWKrJaFBuL5W521YTaLAZoJFqBskAUQi9y4MijxX92G9F"],"license":"Apache-2.0 OR MIT"}},"version":1},"id":43} \ No newline at end of file diff --git a/pdp/contract/PDPVerifier.abi b/pdp/contract/PDPVerifier.abi index 93aabc8db..11c6256d8 100644 --- a/pdp/contract/PDPVerifier.abi +++ b/pdp/contract/PDPVerifier.abi @@ -71,7 +71,7 @@ }, { "type": "function", - "name": "MAX_ROOT_SIZE", + "name": "MAX_PIECE_SIZE", "inputs": [], "outputs": [ { @@ -162,7 +162,7 @@ }, { "type": "function", - "name": "addRoots", + "name": "addPieces", "inputs": [ { "name": "setId", @@ -170,12 +170,12 @@ "internalType": "uint256" }, { - "name": "rootData", + "name": "pieceData", "type": "tuple[]", - "internalType": "struct PDPVerifier.RootData[]", + "internalType": "struct PDPVerifier.PieceData[]", "components": [ { - "name": "root", + "name": "piece", "type": "tuple", "internalType": "struct Cids.Cid", "components": [ @@ -234,12 +234,17 @@ }, { "type": "function", - "name": "claimProofSetOwnership", + "name": "claimDataSetStorageProvider", "inputs": [ { "name": "setId", "type": "uint256", "internalType": "uint256" + }, + { + "name": "extraData", + "type": "bytes", + "internalType": "bytes" } ], "outputs": [], @@ -247,7 +252,7 @@ }, { "type": "function", - "name": "createProofSet", + "name": "createDataSet", "inputs": [ { "name": "listenerAddr", @@ -271,7 +276,7 @@ }, { "type": "function", - "name": "deleteProofSet", + "name": "deleteDataSet", "inputs": [ { "name": "setId", @@ -289,7 +294,7 @@ }, { "type": "function", - "name": "findRootIds", + "name": "findPieceIds", "inputs": [ { "name": "setId", @@ -306,10 +311,10 @@ { "name": "", "type": "tuple[]", - "internalType": "struct PDPVerifier.RootIdAndOffset[]", + "internalType": "struct PDPVerifier.PieceIdAndOffset[]", "components": [ { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" }, @@ -394,7 +399,7 @@ }, { "type": "function", - "name": "getNextProofSetId", + "name": "getNextDataSetId", "inputs": [], "outputs": [ { @@ -407,7 +412,7 @@ }, { "type": "function", - "name": "getNextRootId", + "name": "getNextPieceId", "inputs": [ { "name": "setId", @@ -426,7 +431,7 @@ }, { "type": "function", - "name": "getProofSetLastProvenEpoch", + "name": "getDataSetLastProvenEpoch", "inputs": [ { "name": "setId", @@ -445,7 +450,7 @@ }, { "type": "function", - "name": "getProofSetLeafCount", + "name": "getDataSetLeafCount", "inputs": [ { "name": "setId", @@ -464,7 +469,7 @@ }, { "type": "function", - "name": "getProofSetListener", + "name": "getDataSetListener", "inputs": [ { "name": "setId", @@ -483,7 +488,7 @@ }, { "type": "function", - "name": "getProofSetOwner", + "name": "getDataSetStorageProvider", "inputs": [ { "name": "setId", @@ -526,7 +531,7 @@ }, { "type": "function", - "name": "getRootCid", + "name": "getPieceCid", "inputs": [ { "name": "setId", @@ -534,7 +539,7 @@ "internalType": "uint256" }, { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" } @@ -557,7 +562,7 @@ }, { "type": "function", - "name": "getRootLeafCount", + "name": "getPieceLeafCount", "inputs": [ { "name": "setId", @@ -565,7 +570,7 @@ "internalType": "uint256" }, { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" } @@ -649,7 +654,7 @@ }, { "type": "function", - "name": "proofSetLive", + "name": "dataSetLive", "inputs": [ { "name": "setId", @@ -668,7 +673,7 @@ }, { "type": "function", - "name": "proposeProofSetOwner", + "name": "proposeDataSetStorageProvider", "inputs": [ { "name": "setId", @@ -676,7 +681,7 @@ "internalType": "uint256" }, { - "name": "newOwner", + "name": "newStorageProvider", "type": "address", "internalType": "address" } @@ -736,7 +741,7 @@ }, { "type": "function", - "name": "rootChallengable", + "name": "pieceChallengable", "inputs": [ { "name": "setId", @@ -744,7 +749,7 @@ "internalType": "uint256" }, { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" } @@ -760,7 +765,7 @@ }, { "type": "function", - "name": "rootLive", + "name": "pieceLive", "inputs": [ { "name": "setId", @@ -768,7 +773,7 @@ "internalType": "uint256" }, { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" } @@ -784,7 +789,7 @@ }, { "type": "function", - "name": "scheduleRemovals", + "name": "schedulePieceDeletions", "inputs": [ { "name": "setId", @@ -792,7 +797,7 @@ "internalType": "uint256" }, { - "name": "rootIds", + "name": "pieceIds", "type": "uint256[]", "internalType": "uint256[]" }, @@ -836,6 +841,76 @@ "outputs": [], "stateMutability": "payable" }, + { + "type": "function", + "name": "getActivePieceCount", + "inputs": [ + { + "name": "setId", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getActivePieces", + "inputs": [ + { + "name": "setId", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "offset", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "limit", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "pieces", + "type": "tuple[]", + "internalType": "struct Cids.Cid[]", + "components": [ + { + "name": "data", + "type": "bytes", + "internalType": "bytes" + } + ] + }, + { + "name": "pieceIds", + "type": "uint256[]", + "internalType": "uint256[]" + }, + { + "name": "rawSizes", + "type": "uint256[]", + "internalType": "uint256[]" + }, + { + "name": "hasMore", + "type": "bool", + "internalType": "bool" + } + ], + "stateMutability": "view" + }, { "type": "event", "name": "Debug", @@ -926,10 +1001,10 @@ "name": "challenges", "type": "tuple[]", "indexed": false, - "internalType": "struct PDPVerifier.RootIdAndOffset[]", + "internalType": "struct PDPVerifier.PieceIdAndOffset[]", "components": [ { - "name": "rootId", + "name": "pieceId", "type": "uint256", "internalType": "uint256" }, @@ -976,7 +1051,7 @@ }, { "type": "event", - "name": "ProofSetCreated", + "name": "DataSetCreated", "inputs": [ { "name": "setId", @@ -985,7 +1060,7 @@ "internalType": "uint256" }, { - "name": "owner", + "name": "storageProvider", "type": "address", "indexed": true, "internalType": "address" @@ -995,7 +1070,7 @@ }, { "type": "event", - "name": "ProofSetDeleted", + "name": "DataSetDeleted", "inputs": [ { "name": "setId", @@ -1014,7 +1089,7 @@ }, { "type": "event", - "name": "ProofSetEmpty", + "name": "DataSetEmpty", "inputs": [ { "name": "setId", @@ -1027,7 +1102,7 @@ }, { "type": "event", - "name": "ProofSetOwnerChanged", + "name": "StorageProviderChanged", "inputs": [ { "name": "setId", @@ -1036,13 +1111,13 @@ "internalType": "uint256" }, { - "name": "oldOwner", + "name": "oldStorageProvider", "type": "address", "indexed": true, "internalType": "address" }, { - "name": "newOwner", + "name": "newStorageProvider", "type": "address", "indexed": true, "internalType": "address" @@ -1052,7 +1127,7 @@ }, { "type": "event", - "name": "RootsAdded", + "name": "PiecesAdded", "inputs": [ { "name": "setId", @@ -1061,7 +1136,7 @@ "internalType": "uint256" }, { - "name": "rootIds", + "name": "pieceIds", "type": "uint256[]", "indexed": false, "internalType": "uint256[]" @@ -1071,7 +1146,7 @@ }, { "type": "event", - "name": "RootsRemoved", + "name": "PiecesRemoved", "inputs": [ { "name": "setId", @@ -1080,7 +1155,7 @@ "internalType": "uint256" }, { - "name": "rootIds", + "name": "pieceIds", "type": "uint256[]", "indexed": false, "internalType": "uint256[]" @@ -1197,4 +1272,4 @@ } ] } -] +] \ No newline at end of file diff --git a/pdp/contract/PDPVerifier.json b/pdp/contract/PDPVerifier.json index 4288a174e..6910d48b7 100644 --- a/pdp/contract/PDPVerifier.json +++ b/pdp/contract/PDPVerifier.json @@ -1 +1 @@ -{"abi":[{"type":"constructor","inputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"BURN_ACTOR","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"EXTRA_DATA_MAX_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"FIL_USD_PRICE_FEED_ID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"LEAF_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_ENQUEUED_REMOVALS","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_ROOT_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_CHALLENGE_SCHEDULED","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_PROVEN_EPOCH","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"PYTH","inputs":[],"outputs":[{"name":"","type":"address","internalType":"contract IPyth"}],"stateMutability":"view"},{"type":"function","name":"RANDOMNESS_PRECOMPILE","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"SECONDS_IN_DAY","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"UPGRADE_INTERFACE_VERSION","inputs":[],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"},{"type":"function","name":"addRoots","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootData","type":"tuple[]","internalType":"struct PDPVerifier.RootData[]","components":[{"name":"root","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"rawSize","type":"uint256","internalType":"uint256"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"calculateProofFee","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"estimatedGasFee","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"claimProofSetOwnership","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createProofSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"deleteProofSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findRootIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct PDPVerifier.RootIdAndOffset[]","components":[{"name":"rootId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getFILUSDPrice","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"},{"name":"","type":"int32","internalType":"int32"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextProofSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextRootId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getProofSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getProofSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getProofSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getProofSetOwner","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getRandomness","inputs":[{"name":"epoch","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getRootCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"stateMutability":"view"},{"type":"function","name":"getRootLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"initialize","inputs":[{"name":"_challengeFinality","type":"uint256","internalType":"uint256"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"owner","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"proofSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeProofSetOwner","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newOwner","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct PDPVerifier.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"proxiableUUID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"renounceOwnership","inputs":[],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"rootChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"rootLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"scheduleRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"rootIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"transferOwnership","inputs":[{"name":"newOwner","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"upgradeToAndCall","inputs":[{"name":"newImplementation","type":"address","internalType":"address"},{"name":"data","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"payable"},{"type":"event","name":"Debug","inputs":[{"name":"message","type":"string","indexed":false,"internalType":"string"},{"name":"value","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"Initialized","inputs":[{"name":"version","type":"uint64","indexed":false,"internalType":"uint64"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"OwnershipTransferred","inputs":[{"name":"previousOwner","type":"address","indexed":true,"internalType":"address"},{"name":"newOwner","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct PDPVerifier.RootIdAndOffset[]","components":[{"name":"rootId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"ProofSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"owner","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"ProofSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"ProofSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"ProofSetOwnerChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldOwner","type":"address","indexed":true,"internalType":"address"},{"name":"newOwner","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"RootsAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"rootIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"RootsRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"rootIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"Upgraded","inputs":[{"name":"implementation","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"error","name":"AddressEmptyCode","inputs":[{"name":"target","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967InvalidImplementation","inputs":[{"name":"implementation","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967NonPayable","inputs":[]},{"type":"error","name":"FailedCall","inputs":[]},{"type":"error","name":"IndexedError","inputs":[{"name":"idx","type":"uint256","internalType":"uint256"},{"name":"msg","type":"string","internalType":"string"}]},{"type":"error","name":"InvalidInitialization","inputs":[]},{"type":"error","name":"NotInitializing","inputs":[]},{"type":"error","name":"OwnableInvalidOwner","inputs":[{"name":"owner","type":"address","internalType":"address"}]},{"type":"error","name":"OwnableUnauthorizedAccount","inputs":[{"name":"account","type":"address","internalType":"address"}]},{"type":"error","name":"UUPSUnauthorizedCallContext","inputs":[]},{"type":"error","name":"UUPSUnsupportedProxiableUUID","inputs":[{"name":"slot","type":"bytes32","internalType":"bytes32"}]}],"bytecode":{"object":"0x60a06040523073ffffffffffffffffffffffffffffffffffffffff1660809073ffffffffffffffffffffffffffffffffffffffff1681525034801562000043575f80fd5b50620000546200005a60201b60201c565b620001c4565b5f6200006b6200015e60201b60201c565b9050805f0160089054906101000a900460ff1615620000b6576040517ff92ee8a900000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b67ffffffffffffffff8016815f015f9054906101000a900467ffffffffffffffff1667ffffffffffffffff16146200015b5767ffffffffffffffff815f015f6101000a81548167ffffffffffffffff021916908367ffffffffffffffff1602179055507fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d267ffffffffffffffff604051620001529190620001a9565b60405180910390a15b50565b5f7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00905090565b5f67ffffffffffffffff82169050919050565b620001a38162000185565b82525050565b5f602082019050620001be5f83018462000198565b92915050565b608051617d10620001eb5f395f8181613de901528181613e3e0152613ff80152617d105ff3fe608060405260043610610271575f3560e01c806367e406d51161014e5780639f8cb3bd116100c0578063f2fde38b11610079578063f2fde38b146109c5578063f58f952b146109ed578063f5cac1ba14610a09578063f83758fe14610a45578063faa6716314610a6f578063fe4b84df14610aab57610271565b80639f8cb3bd146108b9578063ad3cb1cc146108e3578063c0e159491461090d578063d49245c114610937578063ee3dac6514610973578063f178b1be1461099b57610271565b806371cf2a161161011257806371cf2a1614610789578063847d1d06146107c557806389208ba9146107ed5780638da5cb5b146108295780638ea417e5146108535780639153e64b1461087d57610271565b806367e406d5146106a95780636ba4608f146106d35780636cb55c161461070f5780636fa4469214610737578063715018a61461077357610271565b80633f84135f116101e757806347331050116101ab57806347331050146105965780634903704a146105d25780634f1ef2861461060e5780634fa279201461062a57806352d1902d1461065557806361a52a361461067f57610271565b80633f84135f1461048f578063453f4f62146104cb57806345c0b92d14610507578063462dd4491461052f5780634726075b1461055957610271565b806315b175701161023957806315b175701461037157806316e2bcd51461039b57806319c75950146103c557806331601226146103ef5780633b68e4e91461042b5780633b7ae9131461045357610271565b8063029b4646146102755780630528a55b1461029f5780630a4d7932146102db5780630a6a63f11461030b57806311c0ee4a14610335575b5f80fd5b348015610280575f80fd5b50610289610ad3565b6040516102969190614f24565b60405180910390f35b3480156102aa575f80fd5b506102c560048036038101906102c09190614fd9565b610ad9565b6040516102d2919061511a565b60405180910390f35b6102f560048036038101906102f091906151e9565b610bc8565b6040516103029190614f24565b60405180910390f35b348015610316575f80fd5b5061031f610ee8565b60405161032c9190615255565b60405180910390f35b348015610340575f80fd5b5061035b600480360381019061035691906152c3565b610f00565b6040516103689190614f24565b60405180910390f35b34801561037c575f80fd5b506103856112a4565b6040516103929190615255565b60405180910390f35b3480156103a6575f80fd5b506103af6112bc565b6040516103bc9190614f24565b60405180910390f35b3480156103d0575f80fd5b506103d96112c7565b6040516103e6919061536c565b60405180910390f35b3480156103fa575f80fd5b5061041560048036038101906104109190615385565b6112ed565b6040516104229190615255565b60405180910390f35b348015610436575f80fd5b50610451600480360381019061044c91906153b0565b61136e565b005b34801561045e575f80fd5b5061047960048036038101906104749190615441565b6116b6565b6040516104869190615530565b60405180910390f35b34801561049a575f80fd5b506104b560048036038101906104b09190615385565b6117c8565b6040516104c29190614f24565b60405180910390f35b3480156104d6575f80fd5b506104f160048036038101906104ec9190615385565b61182a565b6040516104fe9190614f24565b60405180910390f35b348015610512575f80fd5b5061052d60048036038101906105289190615550565b611924565b005b34801561053a575f80fd5b50610543611e04565b6040516105509190614f24565b60405180910390f35b348015610564575f80fd5b5061057f600480360381019061057a9190615385565b611e08565b60405161058d9291906155c1565b60405180910390f35b3480156105a1575f80fd5b506105bc60048036038101906105b79190615441565b611ebd565b6040516105c99190615602565b60405180910390f35b3480156105dd575f80fd5b506105f860048036038101906105f39190615441565b611f16565b6040516106059190614f24565b60405180910390f35b61062860048036038101906106239190615743565b611f7a565b005b348015610635575f80fd5b5061063e611f99565b60405161064c9291906157da565b60405180910390f35b348015610660575f80fd5b506106696120ab565b604051610676919061536c565b60405180910390f35b34801561068a575f80fd5b506106936120dc565b6040516106a09190614f24565b60405180910390f35b3480156106b4575f80fd5b506106bd6120e3565b6040516106ca919061585c565b60405180910390f35b3480156106de575f80fd5b506106f960048036038101906106f49190615385565b6120fb565b6040516107069190614f24565b60405180910390f35b34801561071a575f80fd5b5061073560048036038101906107309190615875565b61215d565b005b348015610742575f80fd5b5061075d60048036038101906107589190615385565b612307565b60405161076a919061595b565b60405180910390f35b34801561077e575f80fd5b50610787612416565b005b348015610794575f80fd5b506107af60048036038101906107aa9190615441565b612429565b6040516107bc9190615602565b60405180910390f35b3480156107d0575f80fd5b506107eb60048036038101906107e6919061597b565b612517565b005b3480156107f8575f80fd5b50610813600480360381019061080e9190615385565b612817565b6040516108209190614f24565b60405180910390f35b348015610834575f80fd5b5061083d612879565b60405161084a9190615255565b60405180910390f35b34801561085e575f80fd5b506108676128ae565b60405161087491906159d8565b60405180910390f35b348015610888575f80fd5b506108a3600480360381019061089e9190615441565b6128ca565b6040516108b09190614f24565b60405180910390f35b3480156108c4575f80fd5b506108cd61293c565b6040516108da9190614f24565b60405180910390f35b3480156108ee575f80fd5b506108f7612942565b6040516109049190615a43565b60405180910390f35b348015610918575f80fd5b5061092161297b565b60405161092e9190614f24565b60405180910390f35b348015610942575f80fd5b5061095d60048036038101906109589190615385565b612980565b60405161096a9190614f24565b60405180910390f35b34801561097e575f80fd5b5061099960048036038101906109949190615385565b6129e2565b005b3480156109a6575f80fd5b506109af612bdd565b6040516109bc9190614f24565b60405180910390f35b3480156109d0575f80fd5b506109eb60048036038101906109e69190615a63565b612be1565b005b610a076004803603810190610a029190615ae3565b612c65565b005b348015610a14575f80fd5b50610a2f6004803603810190610a2a9190615385565b613215565b604051610a3c9190615602565b60405180910390f35b348015610a50575f80fd5b50610a596132a7565b604051610a669190614f24565b60405180910390f35b348015610a7a575f80fd5b50610a956004803603810190610a909190615385565b6132af565b604051610aa29190614f24565b60405180910390f35b348015610ab6575f80fd5b50610ad16004803603810190610acc9190615385565b613311565b005b61080081565b60605f610af660055f8781526020019081526020015f20546134a0565b610100610b039190615b6d565b90505f8484905067ffffffffffffffff811115610b2357610b2261561f565b5b604051908082528060200260200182016040528015610b5c57816020015b610b49614e89565b815260200190600190039081610b415790505b5090505f5b85859050811015610bbb57610b9087878784818110610b8357610b82615ba0565b5b90506020020135856135c9565b828281518110610ba357610ba2615ba0565b5b60200260200101819052508080600101915050610b61565b5080925050509392505050565b5f610800838390501115610c11576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610c0890615c17565b60405180910390fd5b5f610c1a6137d4565b905080341015610c5f576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610c5690615c7f565b60405180910390fd5b610c68816137fb565b80341115610cc0573373ffffffffffffffffffffffffffffffffffffffff166108fc8234610c969190615b6d565b90811502906040515f60405180830381858888f19350505050158015610cbe573d5f803e3d5ffd5b505b5f60015f81819054906101000a900467ffffffffffffffff1680929190610ce690615c9d565b91906101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555067ffffffffffffffff1690505f60065f8381526020019081526020015f20819055505f60075f8381526020019081526020015f208190555033600b5f8381526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055508560085f8381526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505f600d5f8381526020019081526020015f20819055505f73ffffffffffffffffffffffffffffffffffffffff168673ffffffffffffffffffffffffffffffffffffffff1614610e98578573ffffffffffffffffffffffffffffffffffffffff166394d41b36823388886040518563ffffffff1660e01b8152600401610e6a9493929190615d08565b5f604051808303815f87803b158015610e81575f80fd5b505af1158015610e93573d5f803e3d5ffd5b505050505b3373ffffffffffffffffffffffffffffffffffffffff16817f017f0b33d96e8f9968590172013032c2346cf047787a5e17a44b0a1bb3cd0f0160405160405180910390a380925050509392505050565b73ff0000000000000000000000000000000000006381565b5f610800838390501115610f49576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610f4090615c17565b60405180910390fd5b610f5286613215565b610f91576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610f8890615d90565b60405180910390fd5b5f8585905011610fd6576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610fcd90615df8565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8881526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614611074576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161106b90615e60565b60405180910390fd5b5f60055f8881526020019081526020015f205490505f8686905067ffffffffffffffff8111156110a7576110a661561f565b5b6040519080825280602002602001820160405280156110d55781602001602082028036833780820191505090505b5090505f5b878790508110156111845761114b89828a8a858181106110fd576110fc615ba0565b5b905060200281019061110f9190615e8a565b805f019061111d9190615eb1565b8b8b868181106111305761112f615ba0565b5b90506020028101906111429190615e8a565b602001356138fe565b5080836111589190615ed8565b82828151811061116b5761116a615ba0565b5b60200260200101818152505080806001019150506110da565b50877f5ce51a8003915c377679ba533d9dafa0792058b254965697e674272f13f4fdd3826040516111b5919061595b565b60405180910390a25f60085f8a81526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614611295578073ffffffffffffffffffffffffffffffffffffffff166312d5d66f8a858b8b8b8b6040518763ffffffff1660e01b81526004016112679695949392919061612e565b5f604051808303815f87803b15801561127e575f80fd5b505af1158015611290573d5f803e3d5ffd5b505050505b82935050505095945050505050565b73fe0000000000000000000000000000000000000681565b660400000000000081565b7f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e5f1b81565b5f6112f782613215565b611336576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161132d90615d90565b60405180910390fd5b60085f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff169050919050565b6108008282905011156113b6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016113ad90615c17565b60405180910390fd5b6113bf85613215565b6113fe576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016113f590615d90565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8781526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff161461149c576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611493906161f3565b60405180910390fd5b6107d0600a5f8781526020019081526020015f2080549050858590506114c29190615ed8565b1115611503576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016114fa90616281565b60405180910390fd5b5f5b848490508110156115d75760055f8781526020019081526020015f205485858381811061153557611534615ba0565b5b905060200201351061157c576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016115739061630f565b60405180910390fd5b600a5f8781526020019081526020015f208585838181106115a05761159f615ba0565b5b90506020020135908060018154018082558091505060019003905f5260205f20015f90919091909150558080600101915050611505565b505f60085f8781526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146116ae578073ffffffffffffffffffffffffffffffffffffffff16634af7d1d287878787876040518663ffffffff1660e01b8152600401611680959493929190616395565b5f604051808303815f87803b158015611697575f80fd5b505af11580156116a9573d5f803e3d5ffd5b505050505b505050505050565b6116be614ea1565b6116c783613215565b611706576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016116fd90615d90565b60405180910390fd5b60025f8481526020019081526020015f205f8381526020019081526020015f206040518060200160405290815f8201805461174090616409565b80601f016020809104026020016040519081016040528092919081815260200182805461176c90616409565b80156117b75780601f1061178e576101008083540402835291602001916117b7565b820191905f5260205f20905b81548152906001019060200180831161179a57829003601f168201915b505050505081525050905092915050565b5f6117d282613215565b611811576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161180890615d90565b60405180910390fd5b60065f8381526020019081526020015f20549050919050565b5f805f73fe0000000000000000000000000000000000000673ffffffffffffffffffffffffffffffffffffffff16846040516020016118699190616459565b60405160208183030381529060405260405161188591906164ad565b5f60405180830381855afa9150503d805f81146118bd576040519150601f19603f3d011682016040523d82523d5f602084013e6118c2565b606091505b509150915081611907576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016118fe90616533565b60405180910390fd5b8080602001905181019061191b9190616565565b92505050919050565b61080082829050111561196c576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161196390615c17565b60405180910390fd5b600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614611a0a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611a0190616600565b60405180910390fd5b5f60065f8681526020019081526020015f205411611a5d576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611a549061668e565b60405180910390fd5b5f600d5f8681526020019081526020015f205403611a8c5743600d5f8681526020019081526020015f20819055505b5f600a5f8681526020019081526020015f2090505f818054905067ffffffffffffffff811115611abf57611abe61561f565b5b604051908082528060200260200182016040528015611aed5781602001602082028036833780820191505090505b5090505f5b8151811015611b77578260018480549050611b0d9190615b6d565b81548110611b1e57611b1d615ba0565b5b905f5260205f200154828281518110611b3a57611b39615ba0565b5b60200260200101818152505082805480611b5757611b566166ac565b5b600190038181905f5260205f20015f905590558080600101915050611af2565b50611b828682613aab565b857fd22bb0ee05b8ca92312459c76223d3b9bc1bd96fb6c9b18e637ededf92d8117482604051611bb2919061595b565b60405180910390a260065f8781526020019081526020015f205460095f8881526020019081526020015f20819055505f5443611bee9190615ed8565b851015611c30576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611c279061676f565b60405180910390fd5b8460075f8881526020019081526020015f20819055505f60065f8881526020019081526020015f205403611cb857857f323c29bc8d678a5d987b90a321982d10b9a91bcad071a9e445879497bf0e68e760405160405180910390a25f600d5f8881526020019081526020015f20819055505f60075f8881526020019081526020015f20819055505b5f60085f8881526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614611db0578073ffffffffffffffffffffffffffffffffffffffff1663aa27ebcc8860075f8b81526020019081526020015f205460065f8c81526020019081526020015f205489896040518663ffffffff1660e01b8152600401611d8295949392919061678d565b5f604051808303815f87803b158015611d99575f80fd5b505af1158015611dab573d5f803e3d5ffd5b505050505b867fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e8760065f8b81526020019081526020015f2054604051611df39291906167d9565b60405180910390a250505050505050565b5f81565b5f80611e1383613215565b611e52576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611e4990615d90565b60405180910390fd5b600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16600c5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1691509150915091565b5f611ec783613215565b8015611ee3575060055f8481526020019081526020015f205482105b8015611f0e57505f60035f8581526020019081526020015f205f8481526020019081526020015f2054115b905092915050565b5f8060095f8581526020019081526020015f20546020611f369190616800565b90505f80611f42611f99565b91509150611f6f85838386600d5f8c81526020019081526020015f205443611f6a9190615b6d565b613b6a565b935050505092915050565b611f82613de7565b611f8b82613ecd565b611f958282613ed8565b5050565b5f805f73a2aa501b19aff244d90cc15a4cf739d2725b572973ffffffffffffffffffffffffffffffffffffffff1663a4ae35e07f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e5f1b620151806040518363ffffffff1660e01b8152600401612010929190616841565b608060405180830381865afa15801561202b573d5f803e3d5ffd5b505050506040513d601f19601f8201168201806040525081019061204f919061696b565b90505f815f015160070b13612099576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161209090616a06565b60405180910390fd5b805f0151816040015192509250509091565b5f6120b4613ff6565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b905090565b6201518081565b73a2aa501b19aff244d90cc15a4cf739d2725b572981565b5f61210582613215565b612144576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161213b90615d90565b60405180910390fd5b60075f8381526020019081526020015f20549050919050565b61216682613215565b6121a5576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161219c90615d90565b60405180910390fd5b5f600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690503373ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614612247576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161223e90616a94565b60405180910390fd5b8173ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16036122b257600c5f8481526020019081526020015f205f6101000a81549073ffffffffffffffffffffffffffffffffffffffff0219169055612302565b81600c5f8581526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505b505050565b606061231282613215565b612351576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161234890615d90565b60405180910390fd5b5f600a5f8481526020019081526020015f2090505f818054905067ffffffffffffffff8111156123845761238361561f565b5b6040519080825280602002602001820160405280156123b25781602001602082028036833780820191505090505b5090505f5b828054905081101561240b578281815481106123d6576123d5615ba0565b5b905f5260205f2001548282815181106123f2576123f1615ba0565b5b60200260200101818152505080806001019150506123b7565b508092505050919050565b61241e61407d565b6124275f614104565b565b5f8061244560055f8681526020019081526020015f20546134a0565b6101006124529190615b6d565b90505f61247d85600160095f8981526020019081526020015f20546124779190615b6d565b846135c9565b9050600160035f8781526020019081526020015f205f835f015181526020019081526020015f20546124af9190615b6d565b8160200151146124f4576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016124eb90616b22565b60405180910390fd5b6124fe8585611ebd565b801561250d5750805f01518411155b9250505092915050565b61080082829050111561255f576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161255690615c17565b60405180910390fd5b60015f9054906101000a900467ffffffffffffffff1667ffffffffffffffff1683106125c0576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016125b790616b8a565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff161461265e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161265590616c18565b60405180910390fd5b5f60065f8581526020019081526020015f205490505f60065f8681526020019081526020015f20819055505f600b5f8681526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505f60075f8681526020019081526020015f20819055505f600d5f8681526020019081526020015f20819055505f60085f8681526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146127d8578073ffffffffffffffffffffffffffffffffffffffff166326c249e3868487876040518563ffffffff1660e01b81526004016127aa9493929190616c36565b5f604051808303815f87803b1580156127c1575f80fd5b505af11580156127d3573d5f803e3d5ffd5b505050505b847f589e9a441b5bddda77c4ab647b0108764a9cc1a7f655aa9b7bc50b8bdfab8673836040516128089190614f24565b60405180910390a25050505050565b5f61282182613215565b612860576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161285790615d90565b60405180910390fd5b60095f8381526020019081526020015f20549050919050565b5f806128836141d5565b9050805f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1691505090565b5f60015f9054906101000a900467ffffffffffffffff16905090565b5f6128d483613215565b612913576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161290a90615d90565b60405180910390fd5b60035f8481526020019081526020015f205f8381526020019081526020015f2054905092915050565b6107d081565b6040518060400160405280600581526020017f352e302e3000000000000000000000000000000000000000000000000000000081525081565b602081565b5f61298a82613215565b6129c9576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016129c090615d90565b60405180910390fd5b60055f8381526020019081526020015f20549050919050565b6129eb81613215565b612a2a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612a2190615d90565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600c5f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614612ac8576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612abf90616ce4565b60405180910390fd5b5f600b5f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905033600b5f8481526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600c5f8381526020019081526020015f205f6101000a81549073ffffffffffffffffffffffffffffffffffffffff02191690553373ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16837fd3273037b635678293ef0c076bd77af13760e75e12806d1db237616d03c3a76660405160405180910390a45050565b5f81565b612be961407d565b5f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1603612c59575f6040517f1e4fbdf7000000000000000000000000000000000000000000000000000000008152600401612c509190615255565b60405180910390fd5b612c6281614104565b50565b5f5a9050600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614612d07576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612cfe90616600565b60405180910390fd5b5f60075f8681526020019081526020015f2054905080431015612d5f576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612d5690616d4c565b60405180910390fd5b5f8484905011612da4576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612d9b90616db4565b60405180910390fd5b5f8103612de6576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612ddd90616e1c565b60405180910390fd5b5f8484905067ffffffffffffffff811115612e0457612e0361561f565b5b604051908082528060200260200182016040528015612e3d57816020015b612e2a614e89565b815260200190600190039081612e225790505b5090505f612e4a876141fc565b90505f60095f8981526020019081526020015f205490505f612e7c60055f8b81526020019081526020015f20546134a0565b610100612e899190615b6d565b90505f5b888890508167ffffffffffffffff16101561308b575f848b83604051602001612eb893929190616e6e565b60405160208183030381529060405290505f8482805190602001205f1c612edf9190616ed7565b9050612eec8c82866135c9565b878467ffffffffffffffff1681518110612f0957612f08615ba0565b5b60200260200101819052505f612f4e612f498e8a8767ffffffffffffffff1681518110612f3957612f38615ba0565b5b60200260200101515f01516116b6565b61421e565b90505f6130328d8d8767ffffffffffffffff16818110612f7157612f70615ba0565b5b9050602002810190612f839190616f07565b8060200190612f929190616f2e565b808060200260200160405190810160405280939291908181526020018383602002808284375f81840152601f19601f82011690508083019250505050505050838f8f8967ffffffffffffffff16818110612fef57612fee615ba0565b5b90506020028101906130019190616f07565b5f01358c8967ffffffffffffffff168151811061302157613020615ba0565b5b602002602001015160200151614360565b905080613074576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161306b90616fda565b60405180910390fd5b50505050808061308390615c9d565b915050612e8d565b505f610514602061309c8b8b614378565b6130a69190615ed8565b6130b09190616800565b5a886130bc9190615b6d565b6130c69190615ed8565b90506130d28a826143fd565b5f60085f8c81526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146131ba578073ffffffffffffffffffffffffffffffffffffffff1663356de02b8c60065f8f81526020019081526020015f2054888e8e90506040518563ffffffff1660e01b815260040161318c9493929190616ff8565b5f604051808303815f87803b1580156131a3575f80fd5b505af11580156131b5573d5f803e3d5ffd5b505050505b43600d5f8d81526020019081526020015f20819055508a7f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e5087604051613200919061511a565b60405180910390a25050505050505050505050565b5f60015f9054906101000a900467ffffffffffffffff1667ffffffffffffffff16821080156132a057505f73ffffffffffffffffffffffffffffffffffffffff16600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614155b9050919050565b5f8054905090565b5f6132b982613215565b6132f8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016132ef90615d90565b60405180910390fd5b600d5f8381526020019081526020015f20549050919050565b5f61331a61450d565b90505f815f0160089054906101000a900460ff161590505f825f015f9054906101000a900467ffffffffffffffff1690505f808267ffffffffffffffff161480156133625750825b90505f60018367ffffffffffffffff1614801561339557505f3073ffffffffffffffffffffffffffffffffffffffff163b145b9050811580156133a3575080155b156133da576040517ff92ee8a900000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b6001855f015f6101000a81548167ffffffffffffffff021916908367ffffffffffffffff1602179055508315613427576001855f0160086101000a81548160ff0219169083151502179055505b61343033614534565b613438614548565b855f819055508315613498575f855f0160086101000a81548160ff0219169083151502179055507fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2600160405161348f9190617074565b60405180910390a15b505050505050565b5f8061010090505f608084901c90505f81146134c9576080826134c39190615b6d565b91508093505b604084901c90505f81146134ea576040826134e49190615b6d565b91508093505b602084901c90505f811461350b576020826135059190615b6d565b91508093505b601084901c90505f811461352c576010826135269190615b6d565b91508093505b600884901c90505f811461354d576008826135479190615b6d565b91508093505b600484901c90505f811461356e576004826135689190615b6d565b91508093505b600284901c90505f811461358f576002826135899190615b6d565b91508093505b600184901c90505f81146135b3576002826135aa9190615b6d565b925050506135c4565b83826135bf9190615b6d565b925050505b919050565b6135d1614e89565b60065f8581526020019081526020015f20548310613624576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161361b906170d7565b60405180910390fd5b5f6001836001901b6136369190615b6d565b90505f80808590505b5f81111561373b5760055f8981526020019081526020015f205484106136825760018161366c9190615b6d565b6001901b8461367b9190615b6d565b9350613728565b60045f8981526020019081526020015f205f8581526020019081526020015f2054836136ae9190615ed8565b91508682116137085760045f8981526020019081526020015f205f8581526020019081526020015f2054836136e39190615ed8565b92506001816136f29190615b6d565b6001901b846137019190615ed8565b9350613727565b6001816137159190615b6d565b6001901b846137249190615b6d565b93505b5b8080613733906170f5565b91505061363f565b5060045f8881526020019081526020015f205f8481526020019081526020015f2054826137689190615ed8565b90508581116137a75760405180604001604052806001856137899190615ed8565b8152602001828861379a9190615b6d565b81525093505050506137cd565b604051806040016040528084815260200183886137c49190615b6d565b81525093505050505b9392505050565b5f600a6001670de0b6b3a76400006137ec9190616800565b6137f6919061711c565b905090565b8034101561383e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161383590617196565b60405180910390fd5b5f73ff0000000000000000000000000000000000006373ffffffffffffffffffffffffffffffffffffffff1682604051613877906171d7565b5f6040518083038185875af1925050503d805f81146138b1576040519150601f19603f3d011682016040523d82523d5f602084013e6138b6565b606091505b50509050806138fa576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016138f190617235565b60405180910390fd5b5050565b5f8060208361390d9190616ed7565b1461394f57836040517fc7b67cf3000000000000000000000000000000000000000000000000000000008152600401613946919061729d565b60405180910390fd5b5f820361399357836040517fc7b67cf300000000000000000000000000000000000000000000000000000000815260040161398a9190617313565b60405180910390fd5b66040000000000008211156139df57836040517fc7b67cf30000000000000000000000000000000000000000000000000000000081526004016139d69190617389565b60405180910390fd5b5f6020836139ed919061711c565b90505f60055f8881526020019081526020015f205f815480929190613a11906173b5565b919050559050613a22878383614552565b8460025f8981526020019081526020015f205f8381526020019081526020015f208181613a4f91906176fc565b9050508160035f8981526020019081526020015f205f8381526020019081526020015f20819055508160065f8981526020019081526020015f205f828254613a979190615ed8565b925050819055508092505050949350505050565b613ab482613215565b613af3576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613aea90615d90565b60405180910390fd5b5f805b8251811015613b3d57613b2384848381518110613b1657613b15615ba0565b5b60200260200101516145ed565b82613b2e9190615ed8565b91508080600101915050613af6565b508060065f8581526020019081526020015f205f828254613b5e9190615b6d565b92505081905550505050565b5f80861180613b7857505f48145b613bb7576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613bae9061777a565b60405180910390fd5b5f8567ffffffffffffffff1611613c03576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613bfa90617808565b60405180910390fd5b5f8311613c45576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613c3c90617896565b60405180910390fd5b5f808560030b12613cc35784600a613c5d91906179f2565b8667ffffffffffffffff166201518065010000000000613c7d9190616800565b613c879190616800565b613c919190616800565b6001670de0b6b3a7640000613ca69190616800565b6002613cb29190616800565b613cbc919061711c565b9050613d3b565b8567ffffffffffffffff166201518065010000000000613ce39190616800565b613ced9190616800565b85613cf790617a3c565b600a613d0391906179f2565b6001670de0b6b3a7640000613d189190616800565b6002613d249190616800565b613d2e9190616800565b613d38919061711c565b90505b5f848483613d499190616800565b613d539190616800565b90505f6064600583613d659190616800565b613d6f919061711c565b90505f6064600484613d819190616800565b613d8b919061711c565b9050818a10613da0575f945050505050613dde565b808a10613dbe578982613db39190615b6d565b945050505050613dde565b6064600184613dcd9190616800565b613dd7919061711c565b9450505050505b95945050505050565b7f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff163073ffffffffffffffffffffffffffffffffffffffff161480613e9457507f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff16613e7b61467b565b73ffffffffffffffffffffffffffffffffffffffff1614155b15613ecb576040517fe07c8dba00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b613ed561407d565b50565b8173ffffffffffffffffffffffffffffffffffffffff166352d1902d6040518163ffffffff1660e01b8152600401602060405180830381865afa925050508015613f4057506040513d601f19601f82011682018060405250810190613f3d9190617aac565b60015b613f8157816040517f4c9c8ce3000000000000000000000000000000000000000000000000000000008152600401613f789190615255565b60405180910390fd5b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b8114613fe757806040517faa1d49a4000000000000000000000000000000000000000000000000000000008152600401613fde919061536c565b60405180910390fd5b613ff183836146ce565b505050565b7f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff163073ffffffffffffffffffffffffffffffffffffffff161461407b576040517fe07c8dba00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b614085614740565b73ffffffffffffffffffffffffffffffffffffffff166140a3612879565b73ffffffffffffffffffffffffffffffffffffffff1614614102576140c6614740565b6040517f118cdaa70000000000000000000000000000000000000000000000000000000081526004016140f99190615255565b60405180910390fd5b565b5f61410d6141d5565b90505f815f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905082825f015f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055508273ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e060405160405180910390a3505050565b5f7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300905090565b5f61421760075f8481526020019081526020015f205461182a565b9050919050565b5f6020825f0151511015614267576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161425e90617b21565b60405180910390fd5b5f602067ffffffffffffffff8111156142835761428261561f565b5b6040519080825280601f01601f1916602001820160405280156142b55781602001600182028036833780820191505090505b5090505f5b602081101561434d57835f0151816020865f0151516142d99190615b6d565b6142e39190615ed8565b815181106142f4576142f3615ba0565b5b602001015160f81c60f81b82828151811061431257614311615ba0565b5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff191690815f1a90535080806001019150506142ba565b508061435890617b62565b915050919050565b5f8361436d868585614747565b149050949350505050565b5f805f90505f5b848490508110156143f257602085858381811061439f5761439e615ba0565b5b90506020028101906143b19190616f07565b80602001906143c09190616f2e565b90506143cc9190616800565b60406143d89190615ed8565b826143e39190615ed8565b9150808060010191505061437f565b508091505092915050565b5f488261440a9190616800565b90505f60095f8581526020019081526020015f2054602061442b9190616800565b90505f80614437611f99565b915091505f61446585848487600d5f8d81526020019081526020015f2054436144609190615b6d565b613b6a565b9050614470816137fb565b803411156144c8573373ffffffffffffffffffffffffffffffffffffffff166108fc823461449e9190615b6d565b90811502906040515f60405180830381858888f193505050501580156144c6573d5f803e3d5ffd5b505b867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b8285856040516144fc93929190617bc8565b60405180910390a250505050505050565b5f7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00905090565b61453c6147cf565b6145458161480f565b50565b6145506147cf565b565b5f8190505f61456082614893565b90505f8490505f5b828110156145bf575f816001901b856145819190615b6d565b905060045f8981526020019081526020015f205f8281526020019081526020015f2054836145af9190615ed8565b9250508080600101915050614568565b508060045f8881526020019081526020015f205f8681526020019081526020015f2081905550505050505050565b5f8060035f8581526020019081526020015f205f8481526020019081526020015f2054905061461d8484836148b0565b60035f8581526020019081526020015f205f8481526020019081526020015f205f905560025f8581526020019081526020015f205f8481526020019081526020015f205f8082015f61466f9190614eb4565b50508091505092915050565b5f6146a77f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b61496b565b5f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905090565b6146d782614974565b8173ffffffffffffffffffffffffffffffffffffffff167fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b60405160405180910390a25f815111156147335761472d8282614a3d565b5061473c565b61473b614abd565b5b5050565b5f33905090565b5f808390505f5b85518110156147c3575f86828151811061476b5761476a615ba0565b5b602002602001015190505f6002866147839190616ed7565b03614799576147928382614af9565b92506147a6565b6147a38184614af9565b92505b6002856147b3919061711c565b945050808060010191505061474e565b50809150509392505050565b6147d7614b0c565b61480d576040517fd7e6bcf800000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b6148176147cf565b5f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1603614887575f6040517f1e4fbdf700000000000000000000000000000000000000000000000000000000815260040161487e9190615255565b60405180910390fd5b61489081614104565b50565b5f6148a96001836148a49190615ed8565b614b2a565b9050919050565b5f6148cb60055f8681526020019081526020015f20546134a0565b6101006148d89190615b6d565b90505f6148e484614893565b90505b818111158015614907575060055f8681526020019081526020015f205484105b15614964578260045f8781526020019081526020015f205f8681526020019081526020015f205f82825461493b9190615b6d565b92505081905550806001901b846149529190615ed8565b935061495d84614893565b90506148e7565b5050505050565b5f819050919050565b5f8173ffffffffffffffffffffffffffffffffffffffff163b036149cf57806040517f4c9c8ce30000000000000000000000000000000000000000000000000000000081526004016149c69190615255565b60405180910390fd5b806149fb7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b61496b565b5f015f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555050565b60605f808473ffffffffffffffffffffffffffffffffffffffff1684604051614a6691906164ad565b5f60405180830381855af49150503d805f8114614a9e576040519150601f19603f3d011682016040523d82523d5f602084013e614aa3565b606091505b5091509150614ab3858383614d6f565b9250505092915050565b5f341115614af7576040517fb398979f00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b5f614b048383614dfc565b905092915050565b5f614b1561450d565b5f0160089054906101000a900460ff16905090565b5f7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff821115614b8e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401614b8590617c6d565b60405180910390fd5b5f61010090505f83614b9f90617c94565b905083811690505f8114614bbc578180614bb8906170f5565b9250505b5f6fffffffffffffffffffffffffffffffff821614614be557608082614be29190615b6d565b91505b5f77ffffffffffffffff0000000000000000ffffffffffffffff821614614c1657604082614c139190615b6d565b91505b5f7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff821614614c4b57602082614c489190615b6d565b91505b5f7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff821614614c8257601082614c7f9190615b6d565b91505b5f7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff821614614cba57600882614cb79190615b6d565b91505b5f7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f821614614cf357600482614cf09190615b6d565b91505b5f7f3333333333333333333333333333333333333333333333333333333333333333821614614d2c57600282614d299190615b6d565b91505b5f7f5555555555555555555555555555555555555555555555555555555555555555821614614d6557600182614d629190615b6d565b91505b8192505050919050565b606082614d8457614d7f82614e45565b614df4565b5f8251148015614daa57505f8473ffffffffffffffffffffffffffffffffffffffff163b145b15614dec57836040517f9996b315000000000000000000000000000000000000000000000000000000008152600401614de39190615255565b60405180910390fd5b819050614df5565b5b9392505050565b5f825f528160205260205f60405f60025afa614e16575f80fd5b5f5190507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f8116905092915050565b5f81511115614e575780518082602001fd5b6040517fd6bda27500000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b60405180604001604052805f81526020015f81525090565b6040518060200160405280606081525090565b508054614ec090616409565b5f825580601f10614ed15750614eee565b601f0160209004905f5260205f2090810190614eed9190614ef1565b5b50565b5b80821115614f08575f815f905550600101614ef2565b5090565b5f819050919050565b614f1e81614f0c565b82525050565b5f602082019050614f375f830184614f15565b92915050565b5f604051905090565b5f80fd5b5f80fd5b614f5781614f0c565b8114614f61575f80fd5b50565b5f81359050614f7281614f4e565b92915050565b5f80fd5b5f80fd5b5f80fd5b5f8083601f840112614f9957614f98614f78565b5b8235905067ffffffffffffffff811115614fb657614fb5614f7c565b5b602083019150836020820283011115614fd257614fd1614f80565b5b9250929050565b5f805f60408486031215614ff057614fef614f46565b5b5f614ffd86828701614f64565b935050602084013567ffffffffffffffff81111561501e5761501d614f4a565b5b61502a86828701614f84565b92509250509250925092565b5f81519050919050565b5f82825260208201905092915050565b5f819050602082019050919050565b61506881614f0c565b82525050565b604082015f8201516150825f85018261505f565b506020820151615095602085018261505f565b50505050565b5f6150a6838361506e565b60408301905092915050565b5f602082019050919050565b5f6150c882615036565b6150d28185615040565b93506150dd83615050565b805f5b8381101561510d5781516150f4888261509b565b97506150ff836150b2565b9250506001810190506150e0565b5085935050505092915050565b5f6020820190508181035f83015261513281846150be565b905092915050565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6151638261513a565b9050919050565b61517381615159565b811461517d575f80fd5b50565b5f8135905061518e8161516a565b92915050565b5f8083601f8401126151a9576151a8614f78565b5b8235905067ffffffffffffffff8111156151c6576151c5614f7c565b5b6020830191508360018202830111156151e2576151e1614f80565b5b9250929050565b5f805f60408486031215615200576151ff614f46565b5b5f61520d86828701615180565b935050602084013567ffffffffffffffff81111561522e5761522d614f4a565b5b61523a86828701615194565b92509250509250925092565b61524f81615159565b82525050565b5f6020820190506152685f830184615246565b92915050565b5f8083601f84011261528357615282614f78565b5b8235905067ffffffffffffffff8111156152a05761529f614f7c565b5b6020830191508360208202830111156152bc576152bb614f80565b5b9250929050565b5f805f805f606086880312156152dc576152db614f46565b5b5f6152e988828901614f64565b955050602086013567ffffffffffffffff81111561530a57615309614f4a565b5b6153168882890161526e565b9450945050604086013567ffffffffffffffff81111561533957615338614f4a565b5b61534588828901615194565b92509250509295509295909350565b5f819050919050565b61536681615354565b82525050565b5f60208201905061537f5f83018461535d565b92915050565b5f6020828403121561539a57615399614f46565b5b5f6153a784828501614f64565b91505092915050565b5f805f805f606086880312156153c9576153c8614f46565b5b5f6153d688828901614f64565b955050602086013567ffffffffffffffff8111156153f7576153f6614f4a565b5b61540388828901614f84565b9450945050604086013567ffffffffffffffff81111561542657615425614f4a565b5b61543288828901615194565b92509250509295509295909350565b5f806040838503121561545757615456614f46565b5b5f61546485828601614f64565b925050602061547585828601614f64565b9150509250929050565b5f81519050919050565b5f82825260208201905092915050565b5f5b838110156154b657808201518184015260208101905061549b565b5f8484015250505050565b5f601f19601f8301169050919050565b5f6154db8261547f565b6154e58185615489565b93506154f5818560208601615499565b6154fe816154c1565b840191505092915050565b5f602083015f8301518482035f86015261552382826154d1565b9150508091505092915050565b5f6020820190508181035f8301526155488184615509565b905092915050565b5f805f806060858703121561556857615567614f46565b5b5f61557587828801614f64565b945050602061558687828801614f64565b935050604085013567ffffffffffffffff8111156155a7576155a6614f4a565b5b6155b387828801615194565b925092505092959194509250565b5f6040820190506155d45f830185615246565b6155e16020830184615246565b9392505050565b5f8115159050919050565b6155fc816155e8565b82525050565b5f6020820190506156155f8301846155f3565b92915050565b5f80fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b615655826154c1565b810181811067ffffffffffffffff821117156156745761567361561f565b5b80604052505050565b5f615686614f3d565b9050615692828261564c565b919050565b5f67ffffffffffffffff8211156156b1576156b061561f565b5b6156ba826154c1565b9050602081019050919050565b828183375f83830152505050565b5f6156e76156e284615697565b61567d565b9050828152602081018484840111156157035761570261561b565b5b61570e8482856156c7565b509392505050565b5f82601f83011261572a57615729614f78565b5b813561573a8482602086016156d5565b91505092915050565b5f806040838503121561575957615758614f46565b5b5f61576685828601615180565b925050602083013567ffffffffffffffff81111561578757615786614f4a565b5b61579385828601615716565b9150509250929050565b5f67ffffffffffffffff82169050919050565b6157b98161579d565b82525050565b5f8160030b9050919050565b6157d4816157bf565b82525050565b5f6040820190506157ed5f8301856157b0565b6157fa60208301846157cb565b9392505050565b5f819050919050565b5f61582461581f61581a8461513a565b615801565b61513a565b9050919050565b5f6158358261580a565b9050919050565b5f6158468261582b565b9050919050565b6158568161583c565b82525050565b5f60208201905061586f5f83018461584d565b92915050565b5f806040838503121561588b5761588a614f46565b5b5f61589885828601614f64565b92505060206158a985828601615180565b9150509250929050565b5f81519050919050565b5f82825260208201905092915050565b5f819050602082019050919050565b5f6158e7838361505f565b60208301905092915050565b5f602082019050919050565b5f615909826158b3565b61591381856158bd565b935061591e836158cd565b805f5b8381101561594e57815161593588826158dc565b9750615940836158f3565b925050600181019050615921565b5085935050505092915050565b5f6020820190508181035f83015261597381846158ff565b905092915050565b5f805f6040848603121561599257615991614f46565b5b5f61599f86828701614f64565b935050602084013567ffffffffffffffff8111156159c0576159bf614f4a565b5b6159cc86828701615194565b92509250509250925092565b5f6020820190506159eb5f8301846157b0565b92915050565b5f81519050919050565b5f82825260208201905092915050565b5f615a15826159f1565b615a1f81856159fb565b9350615a2f818560208601615499565b615a38816154c1565b840191505092915050565b5f6020820190508181035f830152615a5b8184615a0b565b905092915050565b5f60208284031215615a7857615a77614f46565b5b5f615a8584828501615180565b91505092915050565b5f8083601f840112615aa357615aa2614f78565b5b8235905067ffffffffffffffff811115615ac057615abf614f7c565b5b602083019150836020820283011115615adc57615adb614f80565b5b9250929050565b5f805f60408486031215615afa57615af9614f46565b5b5f615b0786828701614f64565b935050602084013567ffffffffffffffff811115615b2857615b27614f4a565b5b615b3486828701615a8e565b92509250509250925092565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b5f615b7782614f0c565b9150615b8283614f0c565b9250828203905081811115615b9a57615b99615b40565b5b92915050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b7f4578747261206461746120746f6f206c617267650000000000000000000000005f82015250565b5f615c016014836159fb565b9150615c0c82615bcd565b602082019050919050565b5f6020820190508181035f830152615c2e81615bf5565b9050919050565b7f737962696c20666565206e6f74206d65740000000000000000000000000000005f82015250565b5f615c696011836159fb565b9150615c7482615c35565b602082019050919050565b5f6020820190508181035f830152615c9681615c5d565b9050919050565b5f615ca78261579d565b915067ffffffffffffffff8203615cc157615cc0615b40565b5b600182019050919050565b5f82825260208201905092915050565b5f615ce78385615ccc565b9350615cf48385846156c7565b615cfd836154c1565b840190509392505050565b5f606082019050615d1b5f830187614f15565b615d286020830186615246565b8181036040830152615d3b818486615cdc565b905095945050505050565b7f50726f6f6620736574206e6f74206c69766500000000000000000000000000005f82015250565b5f615d7a6012836159fb565b9150615d8582615d46565b602082019050919050565b5f6020820190508181035f830152615da781615d6e565b9050919050565b7f4d75737420616464206174206c65617374206f6e6520726f6f740000000000005f82015250565b5f615de2601a836159fb565b9150615ded82615dae565b602082019050919050565b5f6020820190508181035f830152615e0f81615dd6565b9050919050565b7f4f6e6c7920746865206f776e65722063616e2061646420726f6f7473000000005f82015250565b5f615e4a601c836159fb565b9150615e5582615e16565b602082019050919050565b5f6020820190508181035f830152615e7781615e3e565b9050919050565b5f80fd5b5f80fd5b5f80fd5b5f82356001604003833603038112615ea557615ea4615e7e565b5b80830191505092915050565b5f82356001602003833603038112615ecc57615ecb615e7e565b5b80830191505092915050565b5f615ee282614f0c565b9150615eed83614f0c565b9250828201905080821115615f0557615f04615b40565b5b92915050565b5f82825260208201905092915050565b5f819050919050565b5f80fd5b5f82356001602003833603038112615f4357615f42615f24565b5b82810191505092915050565b5f80fd5b5f80fd5b5f8083356001602003843603038112615f7357615f72615f24565b5b83810192508235915060208301925067ffffffffffffffff821115615f9b57615f9a615f4f565b5b600182023603831315615fb157615fb0615f53565b5b509250929050565b5f615fc48385615489565b9350615fd18385846156c7565b615fda836154c1565b840190509392505050565b5f60208301615ff65f840184615f57565b8583035f870152616008838284615fb9565b925050508091505092915050565b5f6160246020840184614f64565b905092915050565b5f6040830161603d5f840184615f28565b8482035f86015261604e8282615fe5565b91505061605e6020840184616016565b61606b602086018261505f565b508091505092915050565b5f616081838361602c565b905092915050565b5f823560016040038336030381126160a4576160a3615f24565b5b82810191505092915050565b5f602082019050919050565b5f6160c78385615f0b565b9350836020840285016160d984615f1b565b805f5b8781101561611c5784840389526160f38284616089565b6160fd8582616076565b9450616108836160b0565b925060208a019950506001810190506160dc565b50829750879450505050509392505050565b5f6080820190506161415f830189614f15565b61614e6020830188614f15565b81810360408301526161618186886160bc565b90508181036060830152616176818486615cdc565b9050979650505050505050565b7f4f6e6c7920746865206f776e65722063616e207363686564756c652072656d6f5f8201527f76616c206f6620726f6f74730000000000000000000000000000000000000000602082015250565b5f6161dd602c836159fb565b91506161e882616183565b604082019050919050565b5f6020820190508181035f83015261620a816161d1565b9050919050565b7f546f6f206d616e792072656d6f76616c73207761697420666f72206e657874205f8201527f70726f76696e6720706572696f6420746f207363686564756c65000000000000602082015250565b5f61626b603a836159fb565b915061627682616211565b604082019050919050565b5f6020820190508181035f8301526162988161625f565b9050919050565b7f43616e206f6e6c79207363686564756c652072656d6f76616c206f66206578695f8201527f7374696e6720726f6f7473000000000000000000000000000000000000000000602082015250565b5f6162f9602b836159fb565b91506163048261629f565b604082019050919050565b5f6020820190508181035f830152616326816162ed565b9050919050565b5f80fd5b82818337505050565b5f61634583856158bd565b93507f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8311156163785761637761632d565b5b602083029250616389838584616331565b82840190509392505050565b5f6060820190506163a85f830188614f15565b81810360208301526163bb81868861633a565b905081810360408301526163d0818486615cdc565b90509695505050505050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b5f600282049050600182168061642057607f821691505b602082108103616433576164326163dc565b5b50919050565b5f819050919050565b61645361644e82614f0c565b616439565b82525050565b5f6164648284616442565b60208201915081905092915050565b5f81905092915050565b5f6164878261547f565b6164918185616473565b93506164a1818560208601615499565b80840191505092915050565b5f6164b8828461647d565b915081905092915050565b7f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c655f8201527f6400000000000000000000000000000000000000000000000000000000000000602082015250565b5f61651d6021836159fb565b9150616528826164c3565b604082019050919050565b5f6020820190508181035f83015261654a81616511565b9050919050565b5f8151905061655f81614f4e565b92915050565b5f6020828403121561657a57616579614f46565b5b5f61658784828501616551565b91505092915050565b7f6f6e6c7920746865206f776e65722063616e206d6f766520746f206e657874205f8201527f70726f76696e6720706572696f64000000000000000000000000000000000000602082015250565b5f6165ea602e836159fb565b91506165f582616590565b604082019050919050565b5f6020820190508181035f830152616617816165de565b9050919050565b7f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c6561765f8201527f6573206172652061646465640000000000000000000000000000000000000000602082015250565b5f616678602c836159fb565b91506166838261661e565b604082019050919050565b5f6020820190508181035f8301526166a58161666c565b9050919050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603160045260245ffd5b7f6368616c6c656e67652065706f6368206d757374206265206174206c656173745f8201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560208201527f2066757475726500000000000000000000000000000000000000000000000000604082015250565b5f6167596047836159fb565b9150616764826166d9565b606082019050919050565b5f6020820190508181035f8301526167868161674d565b9050919050565b5f6080820190506167a05f830188614f15565b6167ad6020830187614f15565b6167ba6040830186614f15565b81810360608301526167cd818486615cdc565b90509695505050505050565b5f6040820190506167ec5f830185614f15565b6167f96020830184614f15565b9392505050565b5f61680a82614f0c565b915061681583614f0c565b925082820261682381614f0c565b9150828204841483151761683a57616839615b40565b5b5092915050565b5f6040820190506168545f83018561535d565b6168616020830184614f15565b9392505050565b5f80fd5b5f8160070b9050919050565b6168818161686c565b811461688b575f80fd5b50565b5f8151905061689c81616878565b92915050565b6168ab8161579d565b81146168b5575f80fd5b50565b5f815190506168c6816168a2565b92915050565b6168d5816157bf565b81146168df575f80fd5b50565b5f815190506168f0816168cc565b92915050565b5f6080828403121561690b5761690a616868565b5b616915608061567d565b90505f6169248482850161688e565b5f830152506020616937848285016168b8565b602083015250604061694b848285016168e2565b604083015250606061695f84828501616551565b60608301525092915050565b5f608082840312156169805761697f614f46565b5b5f61698d848285016168f6565b91505092915050565b7f6661696c656420746f2076616c69646174653a207072696365206d75737420625f8201527f652067726561746572207468616e203000000000000000000000000000000000602082015250565b5f6169f06030836159fb565b91506169fb82616996565b604082019050919050565b5f6020820190508181035f830152616a1d816169e4565b9050919050565b7f4f6e6c79207468652063757272656e74206f776e65722063616e2070726f706f5f8201527f73652061206e6577206f776e6572000000000000000000000000000000000000602082015250565b5f616a7e602e836159fb565b9150616a8982616a24565b604082019050919050565b5f6020820190508181035f830152616aab81616a72565b9050919050565b7f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e20775f8201527f697468207468652076657279206c617374206c656166206f66206120726f6f74602082015250565b5f616b0c6040836159fb565b9150616b1782616ab2565b604082019050919050565b5f6020820190508181035f830152616b3981616b00565b9050919050565b7f70726f6f6620736574206964206f7574206f6620626f756e64730000000000005f82015250565b5f616b74601a836159fb565b9150616b7f82616b40565b602082019050919050565b5f6020820190508181035f830152616ba181616b68565b9050919050565b7f4f6e6c7920746865206f776e65722063616e2064656c6574652070726f6f66205f8201527f7365747300000000000000000000000000000000000000000000000000000000602082015250565b5f616c026024836159fb565b9150616c0d82616ba8565b604082019050919050565b5f6020820190508181035f830152616c2f81616bf6565b9050919050565b5f606082019050616c495f830187614f15565b616c566020830186614f15565b8181036040830152616c69818486615cdc565b905095945050505050565b7f4f6e6c79207468652070726f706f736564206f776e65722063616e20636c61695f8201527f6d206f776e657273686970000000000000000000000000000000000000000000602082015250565b5f616cce602b836159fb565b9150616cd982616c74565b604082019050919050565b5f6020820190508181035f830152616cfb81616cc2565b9050919050565b7f7072656d61747572652070726f6f6600000000000000000000000000000000005f82015250565b5f616d36600f836159fb565b9150616d4182616d02565b602082019050919050565b5f6020820190508181035f830152616d6381616d2a565b9050919050565b7f656d7074792070726f6f660000000000000000000000000000000000000000005f82015250565b5f616d9e600b836159fb565b9150616da982616d6a565b602082019050919050565b5f6020820190508181035f830152616dcb81616d92565b9050919050565b7f6e6f206368616c6c656e6765207363686564756c6564000000000000000000005f82015250565b5f616e066016836159fb565b9150616e1182616dd2565b602082019050919050565b5f6020820190508181035f830152616e3381616dfa565b9050919050565b5f8160c01b9050919050565b5f616e5082616e3a565b9050919050565b616e68616e638261579d565b616e46565b82525050565b5f616e798286616442565b602082019150616e898285616442565b602082019150616e998284616e57565b600882019150819050949350505050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b5f616ee182614f0c565b9150616eec83614f0c565b925082616efc57616efb616eaa565b5b828206905092915050565b5f82356001604003833603038112616f2257616f21615e7e565b5b80830191505092915050565b5f8083356001602003843603038112616f4a57616f49615e7e565b5b80840192508235915067ffffffffffffffff821115616f6c57616f6b615e82565b5b602083019250602082023603831315616f8857616f87615e86565b5b509250929050565b7f70726f6f6620646964206e6f74207665726966790000000000000000000000005f82015250565b5f616fc46014836159fb565b9150616fcf82616f90565b602082019050919050565b5f6020820190508181035f830152616ff181616fb8565b9050919050565b5f60808201905061700b5f830187614f15565b6170186020830186614f15565b6170256040830185614f15565b6170326060830184614f15565b95945050505050565b5f819050919050565b5f61705e6170596170548461703b565b615801565b61579d565b9050919050565b61706e81617044565b82525050565b5f6020820190506170875f830184617065565b92915050565b7f4c65616620696e646578206f7574206f6620626f756e647300000000000000005f82015250565b5f6170c16018836159fb565b91506170cc8261708d565b602082019050919050565b5f6020820190508181035f8301526170ee816170b5565b9050919050565b5f6170ff82614f0c565b91505f820361711157617110615b40565b5b600182039050919050565b5f61712682614f0c565b915061713183614f0c565b92508261714157617140616eaa565b5b828204905092915050565b7f496e636f72726563742066656520616d6f756e740000000000000000000000005f82015250565b5f6171806014836159fb565b915061718b8261714c565b602082019050919050565b5f6020820190508181035f8301526171ad81617174565b9050919050565b50565b5f6171c25f83616473565b91506171cd826171b4565b5f82019050919050565b5f6171e1826171b7565b9150819050919050565b7f4275726e206661696c65640000000000000000000000000000000000000000005f82015250565b5f61721f600b836159fb565b915061722a826171eb565b602082019050919050565b5f6020820190508181035f83015261724c81617213565b9050919050565b7f53697a65206d7573742062652061206d756c7469706c65206f662033320000005f82015250565b5f617287601d836159fb565b915061729282617253565b602082019050919050565b5f6040820190506172b05f830184614f15565b81810360208301526172c18161727b565b905092915050565b7f53697a65206d7573742062652067726561746572207468616e203000000000005f82015250565b5f6172fd601b836159fb565b9150617308826172c9565b602082019050919050565b5f6040820190506173265f830184614f15565b8181036020830152617337816172f1565b905092915050565b7f526f6f742073697a65206d757374206265206c657373207468616e20325e35305f82015250565b5f6173736020836159fb565b915061737e8261733f565b602082019050919050565b5f60408201905061739c5f830184614f15565b81810360208301526173ad81617367565b905092915050565b5f6173bf82614f0c565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82036173f1576173f0615b40565b5b600182019050919050565b5f808335600160200384360303811261741857617417615e7e565b5b80840192508235915067ffffffffffffffff82111561743a57617439615e82565b5b60208301925060018202360383131561745657617455615e86565b5b509250929050565b5f82905092915050565b5f819050815f5260205f209050919050565b5f6020601f8301049050919050565b5f82821b905092915050565b5f600883026174c47fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82617489565b6174ce8683617489565b95508019841693508086168417925050509392505050565b5f6175006174fb6174f684614f0c565b615801565b614f0c565b9050919050565b5f819050919050565b617519836174e6565b61752d61752582617507565b848454617495565b825550505050565b5f90565b617541617535565b61754c818484617510565b505050565b5b8181101561756f576175645f82617539565b600181019050617552565b5050565b601f8211156175b45761758581617468565b61758e8461747a565b8101602085101561759d578190505b6175b16175a98561747a565b830182617551565b50505b505050565b5f82821c905092915050565b5f6175d45f19846008026175b9565b1980831691505092915050565b5f6175ec83836175c5565b9150826002028217905092915050565b617606838361745e565b67ffffffffffffffff81111561761f5761761e61561f565b5b6176298254616409565b617634828285617573565b5f601f831160018114617661575f841561764f578287013590505b61765985826175e1565b8655506176c0565b601f19841661766f86617468565b5f5b8281101561769657848901358255600182019150602085019450602081019050617671565b868310156176b357848901356176af601f8916826175c5565b8355505b6001600288020188555050505b50505050505050565b6176d48383836175fc565b505050565b5f81015f83016176e981856173fc565b6176f48183866176c9565b505050505050565b61770682826176d9565b5050565b7f6661696c656420746f2076616c69646174653a20657374696d617465642067615f8201527f7320666565206d7573742062652067726561746572207468616e203000000000602082015250565b5f617764603c836159fb565b915061776f8261770a565b604082019050919050565b5f6020820190508181035f83015261779181617758565b9050919050565b7f6661696c656420746f2076616c69646174653a204174746f46494c20707269635f8201527f65206d7573742062652067726561746572207468616e20300000000000000000602082015250565b5f6177f26038836159fb565b91506177fd82617798565b604082019050919050565b5f6020820190508181035f83015261781f816177e6565b9050919050565b7f6661696c656420746f2076616c69646174653a207261772073697a65206d75735f8201527f742062652067726561746572207468616e203000000000000000000000000000602082015250565b5f6178806033836159fb565b915061788b82617826565b604082019050919050565b5f6020820190508181035f8301526178ad81617874565b9050919050565b5f8160011c9050919050565b5f808291508390505b6001851115617909578086048111156178e5576178e4615b40565b5b60018516156178f45780820291505b8081029050617902856178b4565b94506178c9565b94509492505050565b5f8261792157600190506179dc565b8161792e575f90506179dc565b8160018114617944576002811461794e5761797d565b60019150506179dc565b60ff8411156179605761795f615b40565b5b8360020a91508482111561797757617976615b40565b5b506179dc565b5060208310610133831016604e8410600b84101617156179b25782820a9050838111156179ad576179ac615b40565b5b6179dc565b6179bf84848460016178c0565b925090508184048111156179d6576179d5615b40565b5b81810290505b9392505050565b5f63ffffffff82169050919050565b5f6179fc82614f0c565b9150617a07836179e3565b9250617a347fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8484617912565b905092915050565b5f617a46826157bf565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008203617a7857617a77615b40565b5b815f039050919050565b617a8b81615354565b8114617a95575f80fd5b50565b5f81519050617aa681617a82565b92915050565b5f60208284031215617ac157617ac0614f46565b5b5f617ace84828501617a98565b91505092915050565b7f436964206461746120697320746f6f2073686f727400000000000000000000005f82015250565b5f617b0b6015836159fb565b9150617b1682617ad7565b602082019050919050565b5f6020820190508181035f830152617b3881617aff565b9050919050565b5f819050602082019050919050565b5f617b598251615354565b80915050919050565b5f617b6c8261547f565b82617b7684617b3f565b9050617b8181617b4e565b92506020821015617bc157617bbc7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83602003600802617489565b831692505b5050919050565b5f606082019050617bdb5f830186614f15565b617be860208301856157b0565b617bf560408301846157cb565b949350505050565b7f496e7075742065786365656473206d6178696d756d20696e743235362076616c5f8201527f7565000000000000000000000000000000000000000000000000000000000000602082015250565b5f617c576022836159fb565b9150617c6282617bfd565b604082019050919050565b5f6020820190508181035f830152617c8481617c4b565b9050919050565b5f819050919050565b5f617c9e82617c8b565b91507f80000000000000000000000000000000000000000000000000000000000000008203617cd057617ccf615b40565b5b815f03905091905056fea264697066735822122033620393606d94f1c8d591222f06f6214eafe22531657caf6478a039d4cfa51864736f6c63430008170033","sourceMap":"1708:31540:17:-:0;;;1171:4:2;1128:48;;;;;;;;;7213:50:17;;;;;;;;;;7234:22;:20;;;:22;;:::i;:::-;1708:31540;;7711:422:1;7826:30;7859:26;:24;;;:26;;:::i;:::-;7826:59;;7900:1;:15;;;;;;;;;;;;7896:76;;;7938:23;;;;;;;;;;;;;;7896:76;8003:16;7985:34;;:1;:14;;;;;;;;;;;;:34;;;7981:146;;8052:16;8035:1;:14;;;:33;;;;;;;;;;;;;;;;;;8087:29;8099:16;8087:29;;;;;;:::i;:::-;;;;;;;;7981:146;7760:373;7711:422::o;8737:170::-;8795:30;8870:21;8860:31;;8737:170;:::o;7:101:20:-;43:7;83:18;76:5;72:30;61:41;;7:101;;;:::o;114:115::-;199:23;216:5;199:23;:::i;:::-;194:3;187:36;114:115;;:::o;235:218::-;326:4;364:2;353:9;349:18;341:26;;377:69;443:1;432:9;428:17;419:6;377:69;:::i;:::-;235:218;;;;:::o;1708:31540:17:-;;;;;;;;;;;;;;;;;;;;;;;","linkReferences":{}},"deployedBytecode":{"object":"0x608060405260043610610271575f3560e01c806367e406d51161014e5780639f8cb3bd116100c0578063f2fde38b11610079578063f2fde38b146109c5578063f58f952b146109ed578063f5cac1ba14610a09578063f83758fe14610a45578063faa6716314610a6f578063fe4b84df14610aab57610271565b80639f8cb3bd146108b9578063ad3cb1cc146108e3578063c0e159491461090d578063d49245c114610937578063ee3dac6514610973578063f178b1be1461099b57610271565b806371cf2a161161011257806371cf2a1614610789578063847d1d06146107c557806389208ba9146107ed5780638da5cb5b146108295780638ea417e5146108535780639153e64b1461087d57610271565b806367e406d5146106a95780636ba4608f146106d35780636cb55c161461070f5780636fa4469214610737578063715018a61461077357610271565b80633f84135f116101e757806347331050116101ab57806347331050146105965780634903704a146105d25780634f1ef2861461060e5780634fa279201461062a57806352d1902d1461065557806361a52a361461067f57610271565b80633f84135f1461048f578063453f4f62146104cb57806345c0b92d14610507578063462dd4491461052f5780634726075b1461055957610271565b806315b175701161023957806315b175701461037157806316e2bcd51461039b57806319c75950146103c557806331601226146103ef5780633b68e4e91461042b5780633b7ae9131461045357610271565b8063029b4646146102755780630528a55b1461029f5780630a4d7932146102db5780630a6a63f11461030b57806311c0ee4a14610335575b5f80fd5b348015610280575f80fd5b50610289610ad3565b6040516102969190614f24565b60405180910390f35b3480156102aa575f80fd5b506102c560048036038101906102c09190614fd9565b610ad9565b6040516102d2919061511a565b60405180910390f35b6102f560048036038101906102f091906151e9565b610bc8565b6040516103029190614f24565b60405180910390f35b348015610316575f80fd5b5061031f610ee8565b60405161032c9190615255565b60405180910390f35b348015610340575f80fd5b5061035b600480360381019061035691906152c3565b610f00565b6040516103689190614f24565b60405180910390f35b34801561037c575f80fd5b506103856112a4565b6040516103929190615255565b60405180910390f35b3480156103a6575f80fd5b506103af6112bc565b6040516103bc9190614f24565b60405180910390f35b3480156103d0575f80fd5b506103d96112c7565b6040516103e6919061536c565b60405180910390f35b3480156103fa575f80fd5b5061041560048036038101906104109190615385565b6112ed565b6040516104229190615255565b60405180910390f35b348015610436575f80fd5b50610451600480360381019061044c91906153b0565b61136e565b005b34801561045e575f80fd5b5061047960048036038101906104749190615441565b6116b6565b6040516104869190615530565b60405180910390f35b34801561049a575f80fd5b506104b560048036038101906104b09190615385565b6117c8565b6040516104c29190614f24565b60405180910390f35b3480156104d6575f80fd5b506104f160048036038101906104ec9190615385565b61182a565b6040516104fe9190614f24565b60405180910390f35b348015610512575f80fd5b5061052d60048036038101906105289190615550565b611924565b005b34801561053a575f80fd5b50610543611e04565b6040516105509190614f24565b60405180910390f35b348015610564575f80fd5b5061057f600480360381019061057a9190615385565b611e08565b60405161058d9291906155c1565b60405180910390f35b3480156105a1575f80fd5b506105bc60048036038101906105b79190615441565b611ebd565b6040516105c99190615602565b60405180910390f35b3480156105dd575f80fd5b506105f860048036038101906105f39190615441565b611f16565b6040516106059190614f24565b60405180910390f35b61062860048036038101906106239190615743565b611f7a565b005b348015610635575f80fd5b5061063e611f99565b60405161064c9291906157da565b60405180910390f35b348015610660575f80fd5b506106696120ab565b604051610676919061536c565b60405180910390f35b34801561068a575f80fd5b506106936120dc565b6040516106a09190614f24565b60405180910390f35b3480156106b4575f80fd5b506106bd6120e3565b6040516106ca919061585c565b60405180910390f35b3480156106de575f80fd5b506106f960048036038101906106f49190615385565b6120fb565b6040516107069190614f24565b60405180910390f35b34801561071a575f80fd5b5061073560048036038101906107309190615875565b61215d565b005b348015610742575f80fd5b5061075d60048036038101906107589190615385565b612307565b60405161076a919061595b565b60405180910390f35b34801561077e575f80fd5b50610787612416565b005b348015610794575f80fd5b506107af60048036038101906107aa9190615441565b612429565b6040516107bc9190615602565b60405180910390f35b3480156107d0575f80fd5b506107eb60048036038101906107e6919061597b565b612517565b005b3480156107f8575f80fd5b50610813600480360381019061080e9190615385565b612817565b6040516108209190614f24565b60405180910390f35b348015610834575f80fd5b5061083d612879565b60405161084a9190615255565b60405180910390f35b34801561085e575f80fd5b506108676128ae565b60405161087491906159d8565b60405180910390f35b348015610888575f80fd5b506108a3600480360381019061089e9190615441565b6128ca565b6040516108b09190614f24565b60405180910390f35b3480156108c4575f80fd5b506108cd61293c565b6040516108da9190614f24565b60405180910390f35b3480156108ee575f80fd5b506108f7612942565b6040516109049190615a43565b60405180910390f35b348015610918575f80fd5b5061092161297b565b60405161092e9190614f24565b60405180910390f35b348015610942575f80fd5b5061095d60048036038101906109589190615385565b612980565b60405161096a9190614f24565b60405180910390f35b34801561097e575f80fd5b5061099960048036038101906109949190615385565b6129e2565b005b3480156109a6575f80fd5b506109af612bdd565b6040516109bc9190614f24565b60405180910390f35b3480156109d0575f80fd5b506109eb60048036038101906109e69190615a63565b612be1565b005b610a076004803603810190610a029190615ae3565b612c65565b005b348015610a14575f80fd5b50610a2f6004803603810190610a2a9190615385565b613215565b604051610a3c9190615602565b60405180910390f35b348015610a50575f80fd5b50610a596132a7565b604051610a669190614f24565b60405180910390f35b348015610a7a575f80fd5b50610a956004803603810190610a909190615385565b6132af565b604051610aa29190614f24565b60405180910390f35b348015610ab6575f80fd5b50610ad16004803603810190610acc9190615385565b613311565b005b61080081565b60605f610af660055f8781526020019081526020015f20546134a0565b610100610b039190615b6d565b90505f8484905067ffffffffffffffff811115610b2357610b2261561f565b5b604051908082528060200260200182016040528015610b5c57816020015b610b49614e89565b815260200190600190039081610b415790505b5090505f5b85859050811015610bbb57610b9087878784818110610b8357610b82615ba0565b5b90506020020135856135c9565b828281518110610ba357610ba2615ba0565b5b60200260200101819052508080600101915050610b61565b5080925050509392505050565b5f610800838390501115610c11576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610c0890615c17565b60405180910390fd5b5f610c1a6137d4565b905080341015610c5f576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610c5690615c7f565b60405180910390fd5b610c68816137fb565b80341115610cc0573373ffffffffffffffffffffffffffffffffffffffff166108fc8234610c969190615b6d565b90811502906040515f60405180830381858888f19350505050158015610cbe573d5f803e3d5ffd5b505b5f60015f81819054906101000a900467ffffffffffffffff1680929190610ce690615c9d565b91906101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555067ffffffffffffffff1690505f60065f8381526020019081526020015f20819055505f60075f8381526020019081526020015f208190555033600b5f8381526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055508560085f8381526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505f600d5f8381526020019081526020015f20819055505f73ffffffffffffffffffffffffffffffffffffffff168673ffffffffffffffffffffffffffffffffffffffff1614610e98578573ffffffffffffffffffffffffffffffffffffffff166394d41b36823388886040518563ffffffff1660e01b8152600401610e6a9493929190615d08565b5f604051808303815f87803b158015610e81575f80fd5b505af1158015610e93573d5f803e3d5ffd5b505050505b3373ffffffffffffffffffffffffffffffffffffffff16817f017f0b33d96e8f9968590172013032c2346cf047787a5e17a44b0a1bb3cd0f0160405160405180910390a380925050509392505050565b73ff0000000000000000000000000000000000006381565b5f610800838390501115610f49576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610f4090615c17565b60405180910390fd5b610f5286613215565b610f91576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610f8890615d90565b60405180910390fd5b5f8585905011610fd6576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610fcd90615df8565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8881526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614611074576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161106b90615e60565b60405180910390fd5b5f60055f8881526020019081526020015f205490505f8686905067ffffffffffffffff8111156110a7576110a661561f565b5b6040519080825280602002602001820160405280156110d55781602001602082028036833780820191505090505b5090505f5b878790508110156111845761114b89828a8a858181106110fd576110fc615ba0565b5b905060200281019061110f9190615e8a565b805f019061111d9190615eb1565b8b8b868181106111305761112f615ba0565b5b90506020028101906111429190615e8a565b602001356138fe565b5080836111589190615ed8565b82828151811061116b5761116a615ba0565b5b60200260200101818152505080806001019150506110da565b50877f5ce51a8003915c377679ba533d9dafa0792058b254965697e674272f13f4fdd3826040516111b5919061595b565b60405180910390a25f60085f8a81526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614611295578073ffffffffffffffffffffffffffffffffffffffff166312d5d66f8a858b8b8b8b6040518763ffffffff1660e01b81526004016112679695949392919061612e565b5f604051808303815f87803b15801561127e575f80fd5b505af1158015611290573d5f803e3d5ffd5b505050505b82935050505095945050505050565b73fe0000000000000000000000000000000000000681565b660400000000000081565b7f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e5f1b81565b5f6112f782613215565b611336576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161132d90615d90565b60405180910390fd5b60085f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff169050919050565b6108008282905011156113b6576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016113ad90615c17565b60405180910390fd5b6113bf85613215565b6113fe576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016113f590615d90565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8781526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff161461149c576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611493906161f3565b60405180910390fd5b6107d0600a5f8781526020019081526020015f2080549050858590506114c29190615ed8565b1115611503576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016114fa90616281565b60405180910390fd5b5f5b848490508110156115d75760055f8781526020019081526020015f205485858381811061153557611534615ba0565b5b905060200201351061157c576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016115739061630f565b60405180910390fd5b600a5f8781526020019081526020015f208585838181106115a05761159f615ba0565b5b90506020020135908060018154018082558091505060019003905f5260205f20015f90919091909150558080600101915050611505565b505f60085f8781526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146116ae578073ffffffffffffffffffffffffffffffffffffffff16634af7d1d287878787876040518663ffffffff1660e01b8152600401611680959493929190616395565b5f604051808303815f87803b158015611697575f80fd5b505af11580156116a9573d5f803e3d5ffd5b505050505b505050505050565b6116be614ea1565b6116c783613215565b611706576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016116fd90615d90565b60405180910390fd5b60025f8481526020019081526020015f205f8381526020019081526020015f206040518060200160405290815f8201805461174090616409565b80601f016020809104026020016040519081016040528092919081815260200182805461176c90616409565b80156117b75780601f1061178e576101008083540402835291602001916117b7565b820191905f5260205f20905b81548152906001019060200180831161179a57829003601f168201915b505050505081525050905092915050565b5f6117d282613215565b611811576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161180890615d90565b60405180910390fd5b60065f8381526020019081526020015f20549050919050565b5f805f73fe0000000000000000000000000000000000000673ffffffffffffffffffffffffffffffffffffffff16846040516020016118699190616459565b60405160208183030381529060405260405161188591906164ad565b5f60405180830381855afa9150503d805f81146118bd576040519150601f19603f3d011682016040523d82523d5f602084013e6118c2565b606091505b509150915081611907576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016118fe90616533565b60405180910390fd5b8080602001905181019061191b9190616565565b92505050919050565b61080082829050111561196c576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161196390615c17565b60405180910390fd5b600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614611a0a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611a0190616600565b60405180910390fd5b5f60065f8681526020019081526020015f205411611a5d576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611a549061668e565b60405180910390fd5b5f600d5f8681526020019081526020015f205403611a8c5743600d5f8681526020019081526020015f20819055505b5f600a5f8681526020019081526020015f2090505f818054905067ffffffffffffffff811115611abf57611abe61561f565b5b604051908082528060200260200182016040528015611aed5781602001602082028036833780820191505090505b5090505f5b8151811015611b77578260018480549050611b0d9190615b6d565b81548110611b1e57611b1d615ba0565b5b905f5260205f200154828281518110611b3a57611b39615ba0565b5b60200260200101818152505082805480611b5757611b566166ac565b5b600190038181905f5260205f20015f905590558080600101915050611af2565b50611b828682613aab565b857fd22bb0ee05b8ca92312459c76223d3b9bc1bd96fb6c9b18e637ededf92d8117482604051611bb2919061595b565b60405180910390a260065f8781526020019081526020015f205460095f8881526020019081526020015f20819055505f5443611bee9190615ed8565b851015611c30576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611c279061676f565b60405180910390fd5b8460075f8881526020019081526020015f20819055505f60065f8881526020019081526020015f205403611cb857857f323c29bc8d678a5d987b90a321982d10b9a91bcad071a9e445879497bf0e68e760405160405180910390a25f600d5f8881526020019081526020015f20819055505f60075f8881526020019081526020015f20819055505b5f60085f8881526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614611db0578073ffffffffffffffffffffffffffffffffffffffff1663aa27ebcc8860075f8b81526020019081526020015f205460065f8c81526020019081526020015f205489896040518663ffffffff1660e01b8152600401611d8295949392919061678d565b5f604051808303815f87803b158015611d99575f80fd5b505af1158015611dab573d5f803e3d5ffd5b505050505b867fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e8760065f8b81526020019081526020015f2054604051611df39291906167d9565b60405180910390a250505050505050565b5f81565b5f80611e1383613215565b611e52576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401611e4990615d90565b60405180910390fd5b600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16600c5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1691509150915091565b5f611ec783613215565b8015611ee3575060055f8481526020019081526020015f205482105b8015611f0e57505f60035f8581526020019081526020015f205f8481526020019081526020015f2054115b905092915050565b5f8060095f8581526020019081526020015f20546020611f369190616800565b90505f80611f42611f99565b91509150611f6f85838386600d5f8c81526020019081526020015f205443611f6a9190615b6d565b613b6a565b935050505092915050565b611f82613de7565b611f8b82613ecd565b611f958282613ed8565b5050565b5f805f73a2aa501b19aff244d90cc15a4cf739d2725b572973ffffffffffffffffffffffffffffffffffffffff1663a4ae35e07f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e5f1b620151806040518363ffffffff1660e01b8152600401612010929190616841565b608060405180830381865afa15801561202b573d5f803e3d5ffd5b505050506040513d601f19601f8201168201806040525081019061204f919061696b565b90505f815f015160070b13612099576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161209090616a06565b60405180910390fd5b805f0151816040015192509250509091565b5f6120b4613ff6565b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b905090565b6201518081565b73a2aa501b19aff244d90cc15a4cf739d2725b572981565b5f61210582613215565b612144576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161213b90615d90565b60405180910390fd5b60075f8381526020019081526020015f20549050919050565b61216682613215565b6121a5576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161219c90615d90565b60405180910390fd5b5f600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690503373ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1614612247576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161223e90616a94565b60405180910390fd5b8173ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16036122b257600c5f8481526020019081526020015f205f6101000a81549073ffffffffffffffffffffffffffffffffffffffff0219169055612302565b81600c5f8581526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505b505050565b606061231282613215565b612351576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161234890615d90565b60405180910390fd5b5f600a5f8481526020019081526020015f2090505f818054905067ffffffffffffffff8111156123845761238361561f565b5b6040519080825280602002602001820160405280156123b25781602001602082028036833780820191505090505b5090505f5b828054905081101561240b578281815481106123d6576123d5615ba0565b5b905f5260205f2001548282815181106123f2576123f1615ba0565b5b60200260200101818152505080806001019150506123b7565b508092505050919050565b61241e61407d565b6124275f614104565b565b5f8061244560055f8681526020019081526020015f20546134a0565b6101006124529190615b6d565b90505f61247d85600160095f8981526020019081526020015f20546124779190615b6d565b846135c9565b9050600160035f8781526020019081526020015f205f835f015181526020019081526020015f20546124af9190615b6d565b8160200151146124f4576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016124eb90616b22565b60405180910390fd5b6124fe8585611ebd565b801561250d5750805f01518411155b9250505092915050565b61080082829050111561255f576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161255690615c17565b60405180910390fd5b60015f9054906101000a900467ffffffffffffffff1667ffffffffffffffff1683106125c0576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016125b790616b8a565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff161461265e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161265590616c18565b60405180910390fd5b5f60065f8581526020019081526020015f205490505f60065f8681526020019081526020015f20819055505f600b5f8681526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505f60075f8681526020019081526020015f20819055505f600d5f8681526020019081526020015f20819055505f60085f8681526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146127d8578073ffffffffffffffffffffffffffffffffffffffff166326c249e3868487876040518563ffffffff1660e01b81526004016127aa9493929190616c36565b5f604051808303815f87803b1580156127c1575f80fd5b505af11580156127d3573d5f803e3d5ffd5b505050505b847f589e9a441b5bddda77c4ab647b0108764a9cc1a7f655aa9b7bc50b8bdfab8673836040516128089190614f24565b60405180910390a25050505050565b5f61282182613215565b612860576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161285790615d90565b60405180910390fd5b60095f8381526020019081526020015f20549050919050565b5f806128836141d5565b9050805f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1691505090565b5f60015f9054906101000a900467ffffffffffffffff16905090565b5f6128d483613215565b612913576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161290a90615d90565b60405180910390fd5b60035f8481526020019081526020015f205f8381526020019081526020015f2054905092915050565b6107d081565b6040518060400160405280600581526020017f352e302e3000000000000000000000000000000000000000000000000000000081525081565b602081565b5f61298a82613215565b6129c9576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016129c090615d90565b60405180910390fd5b60055f8381526020019081526020015f20549050919050565b6129eb81613215565b612a2a576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612a2190615d90565b60405180910390fd5b3373ffffffffffffffffffffffffffffffffffffffff16600c5f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614612ac8576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612abf90616ce4565b60405180910390fd5b5f600b5f8381526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905033600b5f8481526020019081526020015f205f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600c5f8381526020019081526020015f205f6101000a81549073ffffffffffffffffffffffffffffffffffffffff02191690553373ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16837fd3273037b635678293ef0c076bd77af13760e75e12806d1db237616d03c3a76660405160405180910390a45050565b5f81565b612be961407d565b5f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1603612c59575f6040517f1e4fbdf7000000000000000000000000000000000000000000000000000000008152600401612c509190615255565b60405180910390fd5b612c6281614104565b50565b5f5a9050600b5f8581526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614612d07576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612cfe90616600565b60405180910390fd5b5f60075f8681526020019081526020015f2054905080431015612d5f576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612d5690616d4c565b60405180910390fd5b5f8484905011612da4576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612d9b90616db4565b60405180910390fd5b5f8103612de6576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401612ddd90616e1c565b60405180910390fd5b5f8484905067ffffffffffffffff811115612e0457612e0361561f565b5b604051908082528060200260200182016040528015612e3d57816020015b612e2a614e89565b815260200190600190039081612e225790505b5090505f612e4a876141fc565b90505f60095f8981526020019081526020015f205490505f612e7c60055f8b81526020019081526020015f20546134a0565b610100612e899190615b6d565b90505f5b888890508167ffffffffffffffff16101561308b575f848b83604051602001612eb893929190616e6e565b60405160208183030381529060405290505f8482805190602001205f1c612edf9190616ed7565b9050612eec8c82866135c9565b878467ffffffffffffffff1681518110612f0957612f08615ba0565b5b60200260200101819052505f612f4e612f498e8a8767ffffffffffffffff1681518110612f3957612f38615ba0565b5b60200260200101515f01516116b6565b61421e565b90505f6130328d8d8767ffffffffffffffff16818110612f7157612f70615ba0565b5b9050602002810190612f839190616f07565b8060200190612f929190616f2e565b808060200260200160405190810160405280939291908181526020018383602002808284375f81840152601f19601f82011690508083019250505050505050838f8f8967ffffffffffffffff16818110612fef57612fee615ba0565b5b90506020028101906130019190616f07565b5f01358c8967ffffffffffffffff168151811061302157613020615ba0565b5b602002602001015160200151614360565b905080613074576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161306b90616fda565b60405180910390fd5b50505050808061308390615c9d565b915050612e8d565b505f610514602061309c8b8b614378565b6130a69190615ed8565b6130b09190616800565b5a886130bc9190615b6d565b6130c69190615ed8565b90506130d28a826143fd565b5f60085f8c81526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690505f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16146131ba578073ffffffffffffffffffffffffffffffffffffffff1663356de02b8c60065f8f81526020019081526020015f2054888e8e90506040518563ffffffff1660e01b815260040161318c9493929190616ff8565b5f604051808303815f87803b1580156131a3575f80fd5b505af11580156131b5573d5f803e3d5ffd5b505050505b43600d5f8d81526020019081526020015f20819055508a7f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e5087604051613200919061511a565b60405180910390a25050505050505050505050565b5f60015f9054906101000a900467ffffffffffffffff1667ffffffffffffffff16821080156132a057505f73ffffffffffffffffffffffffffffffffffffffff16600b5f8481526020019081526020015f205f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1614155b9050919050565b5f8054905090565b5f6132b982613215565b6132f8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016132ef90615d90565b60405180910390fd5b600d5f8381526020019081526020015f20549050919050565b5f61331a61450d565b90505f815f0160089054906101000a900460ff161590505f825f015f9054906101000a900467ffffffffffffffff1690505f808267ffffffffffffffff161480156133625750825b90505f60018367ffffffffffffffff1614801561339557505f3073ffffffffffffffffffffffffffffffffffffffff163b145b9050811580156133a3575080155b156133da576040517ff92ee8a900000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b6001855f015f6101000a81548167ffffffffffffffff021916908367ffffffffffffffff1602179055508315613427576001855f0160086101000a81548160ff0219169083151502179055505b61343033614534565b613438614548565b855f819055508315613498575f855f0160086101000a81548160ff0219169083151502179055507fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2600160405161348f9190617074565b60405180910390a15b505050505050565b5f8061010090505f608084901c90505f81146134c9576080826134c39190615b6d565b91508093505b604084901c90505f81146134ea576040826134e49190615b6d565b91508093505b602084901c90505f811461350b576020826135059190615b6d565b91508093505b601084901c90505f811461352c576010826135269190615b6d565b91508093505b600884901c90505f811461354d576008826135479190615b6d565b91508093505b600484901c90505f811461356e576004826135689190615b6d565b91508093505b600284901c90505f811461358f576002826135899190615b6d565b91508093505b600184901c90505f81146135b3576002826135aa9190615b6d565b925050506135c4565b83826135bf9190615b6d565b925050505b919050565b6135d1614e89565b60065f8581526020019081526020015f20548310613624576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161361b906170d7565b60405180910390fd5b5f6001836001901b6136369190615b6d565b90505f80808590505b5f81111561373b5760055f8981526020019081526020015f205484106136825760018161366c9190615b6d565b6001901b8461367b9190615b6d565b9350613728565b60045f8981526020019081526020015f205f8581526020019081526020015f2054836136ae9190615ed8565b91508682116137085760045f8981526020019081526020015f205f8581526020019081526020015f2054836136e39190615ed8565b92506001816136f29190615b6d565b6001901b846137019190615ed8565b9350613727565b6001816137159190615b6d565b6001901b846137249190615b6d565b93505b5b8080613733906170f5565b91505061363f565b5060045f8881526020019081526020015f205f8481526020019081526020015f2054826137689190615ed8565b90508581116137a75760405180604001604052806001856137899190615ed8565b8152602001828861379a9190615b6d565b81525093505050506137cd565b604051806040016040528084815260200183886137c49190615b6d565b81525093505050505b9392505050565b5f600a6001670de0b6b3a76400006137ec9190616800565b6137f6919061711c565b905090565b8034101561383e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161383590617196565b60405180910390fd5b5f73ff0000000000000000000000000000000000006373ffffffffffffffffffffffffffffffffffffffff1682604051613877906171d7565b5f6040518083038185875af1925050503d805f81146138b1576040519150601f19603f3d011682016040523d82523d5f602084013e6138b6565b606091505b50509050806138fa576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016138f190617235565b60405180910390fd5b5050565b5f8060208361390d9190616ed7565b1461394f57836040517fc7b67cf3000000000000000000000000000000000000000000000000000000008152600401613946919061729d565b60405180910390fd5b5f820361399357836040517fc7b67cf300000000000000000000000000000000000000000000000000000000815260040161398a9190617313565b60405180910390fd5b66040000000000008211156139df57836040517fc7b67cf30000000000000000000000000000000000000000000000000000000081526004016139d69190617389565b60405180910390fd5b5f6020836139ed919061711c565b90505f60055f8881526020019081526020015f205f815480929190613a11906173b5565b919050559050613a22878383614552565b8460025f8981526020019081526020015f205f8381526020019081526020015f208181613a4f91906176fc565b9050508160035f8981526020019081526020015f205f8381526020019081526020015f20819055508160065f8981526020019081526020015f205f828254613a979190615ed8565b925050819055508092505050949350505050565b613ab482613215565b613af3576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613aea90615d90565b60405180910390fd5b5f805b8251811015613b3d57613b2384848381518110613b1657613b15615ba0565b5b60200260200101516145ed565b82613b2e9190615ed8565b91508080600101915050613af6565b508060065f8581526020019081526020015f205f828254613b5e9190615b6d565b92505081905550505050565b5f80861180613b7857505f48145b613bb7576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613bae9061777a565b60405180910390fd5b5f8567ffffffffffffffff1611613c03576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613bfa90617808565b60405180910390fd5b5f8311613c45576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401613c3c90617896565b60405180910390fd5b5f808560030b12613cc35784600a613c5d91906179f2565b8667ffffffffffffffff166201518065010000000000613c7d9190616800565b613c879190616800565b613c919190616800565b6001670de0b6b3a7640000613ca69190616800565b6002613cb29190616800565b613cbc919061711c565b9050613d3b565b8567ffffffffffffffff166201518065010000000000613ce39190616800565b613ced9190616800565b85613cf790617a3c565b600a613d0391906179f2565b6001670de0b6b3a7640000613d189190616800565b6002613d249190616800565b613d2e9190616800565b613d38919061711c565b90505b5f848483613d499190616800565b613d539190616800565b90505f6064600583613d659190616800565b613d6f919061711c565b90505f6064600484613d819190616800565b613d8b919061711c565b9050818a10613da0575f945050505050613dde565b808a10613dbe578982613db39190615b6d565b945050505050613dde565b6064600184613dcd9190616800565b613dd7919061711c565b9450505050505b95945050505050565b7f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff163073ffffffffffffffffffffffffffffffffffffffff161480613e9457507f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff16613e7b61467b565b73ffffffffffffffffffffffffffffffffffffffff1614155b15613ecb576040517fe07c8dba00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b613ed561407d565b50565b8173ffffffffffffffffffffffffffffffffffffffff166352d1902d6040518163ffffffff1660e01b8152600401602060405180830381865afa925050508015613f4057506040513d601f19601f82011682018060405250810190613f3d9190617aac565b60015b613f8157816040517f4c9c8ce3000000000000000000000000000000000000000000000000000000008152600401613f789190615255565b60405180910390fd5b7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b8114613fe757806040517faa1d49a4000000000000000000000000000000000000000000000000000000008152600401613fde919061536c565b60405180910390fd5b613ff183836146ce565b505050565b7f000000000000000000000000000000000000000000000000000000000000000073ffffffffffffffffffffffffffffffffffffffff163073ffffffffffffffffffffffffffffffffffffffff161461407b576040517fe07c8dba00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b614085614740565b73ffffffffffffffffffffffffffffffffffffffff166140a3612879565b73ffffffffffffffffffffffffffffffffffffffff1614614102576140c6614740565b6040517f118cdaa70000000000000000000000000000000000000000000000000000000081526004016140f99190615255565b60405180910390fd5b565b5f61410d6141d5565b90505f815f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905082825f015f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055508273ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e060405160405180910390a3505050565b5f7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300905090565b5f61421760075f8481526020019081526020015f205461182a565b9050919050565b5f6020825f0151511015614267576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161425e90617b21565b60405180910390fd5b5f602067ffffffffffffffff8111156142835761428261561f565b5b6040519080825280601f01601f1916602001820160405280156142b55781602001600182028036833780820191505090505b5090505f5b602081101561434d57835f0151816020865f0151516142d99190615b6d565b6142e39190615ed8565b815181106142f4576142f3615ba0565b5b602001015160f81c60f81b82828151811061431257614311615ba0565b5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff191690815f1a90535080806001019150506142ba565b508061435890617b62565b915050919050565b5f8361436d868585614747565b149050949350505050565b5f805f90505f5b848490508110156143f257602085858381811061439f5761439e615ba0565b5b90506020028101906143b19190616f07565b80602001906143c09190616f2e565b90506143cc9190616800565b60406143d89190615ed8565b826143e39190615ed8565b9150808060010191505061437f565b508091505092915050565b5f488261440a9190616800565b90505f60095f8581526020019081526020015f2054602061442b9190616800565b90505f80614437611f99565b915091505f61446585848487600d5f8d81526020019081526020015f2054436144609190615b6d565b613b6a565b9050614470816137fb565b803411156144c8573373ffffffffffffffffffffffffffffffffffffffff166108fc823461449e9190615b6d565b90811502906040515f60405180830381858888f193505050501580156144c6573d5f803e3d5ffd5b505b867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b8285856040516144fc93929190617bc8565b60405180910390a250505050505050565b5f7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00905090565b61453c6147cf565b6145458161480f565b50565b6145506147cf565b565b5f8190505f61456082614893565b90505f8490505f5b828110156145bf575f816001901b856145819190615b6d565b905060045f8981526020019081526020015f205f8281526020019081526020015f2054836145af9190615ed8565b9250508080600101915050614568565b508060045f8881526020019081526020015f205f8681526020019081526020015f2081905550505050505050565b5f8060035f8581526020019081526020015f205f8481526020019081526020015f2054905061461d8484836148b0565b60035f8581526020019081526020015f205f8481526020019081526020015f205f905560025f8581526020019081526020015f205f8481526020019081526020015f205f8082015f61466f9190614eb4565b50508091505092915050565b5f6146a77f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b61496b565b5f015f9054906101000a900473ffffffffffffffffffffffffffffffffffffffff16905090565b6146d782614974565b8173ffffffffffffffffffffffffffffffffffffffff167fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b60405160405180910390a25f815111156147335761472d8282614a3d565b5061473c565b61473b614abd565b5b5050565b5f33905090565b5f808390505f5b85518110156147c3575f86828151811061476b5761476a615ba0565b5b602002602001015190505f6002866147839190616ed7565b03614799576147928382614af9565b92506147a6565b6147a38184614af9565b92505b6002856147b3919061711c565b945050808060010191505061474e565b50809150509392505050565b6147d7614b0c565b61480d576040517fd7e6bcf800000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b6148176147cf565b5f73ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff1603614887575f6040517f1e4fbdf700000000000000000000000000000000000000000000000000000000815260040161487e9190615255565b60405180910390fd5b61489081614104565b50565b5f6148a96001836148a49190615ed8565b614b2a565b9050919050565b5f6148cb60055f8681526020019081526020015f20546134a0565b6101006148d89190615b6d565b90505f6148e484614893565b90505b818111158015614907575060055f8681526020019081526020015f205484105b15614964578260045f8781526020019081526020015f205f8681526020019081526020015f205f82825461493b9190615b6d565b92505081905550806001901b846149529190615ed8565b935061495d84614893565b90506148e7565b5050505050565b5f819050919050565b5f8173ffffffffffffffffffffffffffffffffffffffff163b036149cf57806040517f4c9c8ce30000000000000000000000000000000000000000000000000000000081526004016149c69190615255565b60405180910390fd5b806149fb7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc5f1b61496b565b5f015f6101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555050565b60605f808473ffffffffffffffffffffffffffffffffffffffff1684604051614a6691906164ad565b5f60405180830381855af49150503d805f8114614a9e576040519150601f19603f3d011682016040523d82523d5f602084013e614aa3565b606091505b5091509150614ab3858383614d6f565b9250505092915050565b5f341115614af7576040517fb398979f00000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b565b5f614b048383614dfc565b905092915050565b5f614b1561450d565b5f0160089054906101000a900460ff16905090565b5f7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff821115614b8e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401614b8590617c6d565b60405180910390fd5b5f61010090505f83614b9f90617c94565b905083811690505f8114614bbc578180614bb8906170f5565b9250505b5f6fffffffffffffffffffffffffffffffff821614614be557608082614be29190615b6d565b91505b5f77ffffffffffffffff0000000000000000ffffffffffffffff821614614c1657604082614c139190615b6d565b91505b5f7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff821614614c4b57602082614c489190615b6d565b91505b5f7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff821614614c8257601082614c7f9190615b6d565b91505b5f7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff821614614cba57600882614cb79190615b6d565b91505b5f7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f821614614cf357600482614cf09190615b6d565b91505b5f7f3333333333333333333333333333333333333333333333333333333333333333821614614d2c57600282614d299190615b6d565b91505b5f7f5555555555555555555555555555555555555555555555555555555555555555821614614d6557600182614d629190615b6d565b91505b8192505050919050565b606082614d8457614d7f82614e45565b614df4565b5f8251148015614daa57505f8473ffffffffffffffffffffffffffffffffffffffff163b145b15614dec57836040517f9996b315000000000000000000000000000000000000000000000000000000008152600401614de39190615255565b60405180910390fd5b819050614df5565b5b9392505050565b5f825f528160205260205f60405f60025afa614e16575f80fd5b5f5190507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f8116905092915050565b5f81511115614e575780518082602001fd5b6040517fd6bda27500000000000000000000000000000000000000000000000000000000815260040160405180910390fd5b60405180604001604052805f81526020015f81525090565b6040518060200160405280606081525090565b508054614ec090616409565b5f825580601f10614ed15750614eee565b601f0160209004905f5260205f2090810190614eed9190614ef1565b5b50565b5b80821115614f08575f815f905550600101614ef2565b5090565b5f819050919050565b614f1e81614f0c565b82525050565b5f602082019050614f375f830184614f15565b92915050565b5f604051905090565b5f80fd5b5f80fd5b614f5781614f0c565b8114614f61575f80fd5b50565b5f81359050614f7281614f4e565b92915050565b5f80fd5b5f80fd5b5f80fd5b5f8083601f840112614f9957614f98614f78565b5b8235905067ffffffffffffffff811115614fb657614fb5614f7c565b5b602083019150836020820283011115614fd257614fd1614f80565b5b9250929050565b5f805f60408486031215614ff057614fef614f46565b5b5f614ffd86828701614f64565b935050602084013567ffffffffffffffff81111561501e5761501d614f4a565b5b61502a86828701614f84565b92509250509250925092565b5f81519050919050565b5f82825260208201905092915050565b5f819050602082019050919050565b61506881614f0c565b82525050565b604082015f8201516150825f85018261505f565b506020820151615095602085018261505f565b50505050565b5f6150a6838361506e565b60408301905092915050565b5f602082019050919050565b5f6150c882615036565b6150d28185615040565b93506150dd83615050565b805f5b8381101561510d5781516150f4888261509b565b97506150ff836150b2565b9250506001810190506150e0565b5085935050505092915050565b5f6020820190508181035f83015261513281846150be565b905092915050565b5f73ffffffffffffffffffffffffffffffffffffffff82169050919050565b5f6151638261513a565b9050919050565b61517381615159565b811461517d575f80fd5b50565b5f8135905061518e8161516a565b92915050565b5f8083601f8401126151a9576151a8614f78565b5b8235905067ffffffffffffffff8111156151c6576151c5614f7c565b5b6020830191508360018202830111156151e2576151e1614f80565b5b9250929050565b5f805f60408486031215615200576151ff614f46565b5b5f61520d86828701615180565b935050602084013567ffffffffffffffff81111561522e5761522d614f4a565b5b61523a86828701615194565b92509250509250925092565b61524f81615159565b82525050565b5f6020820190506152685f830184615246565b92915050565b5f8083601f84011261528357615282614f78565b5b8235905067ffffffffffffffff8111156152a05761529f614f7c565b5b6020830191508360208202830111156152bc576152bb614f80565b5b9250929050565b5f805f805f606086880312156152dc576152db614f46565b5b5f6152e988828901614f64565b955050602086013567ffffffffffffffff81111561530a57615309614f4a565b5b6153168882890161526e565b9450945050604086013567ffffffffffffffff81111561533957615338614f4a565b5b61534588828901615194565b92509250509295509295909350565b5f819050919050565b61536681615354565b82525050565b5f60208201905061537f5f83018461535d565b92915050565b5f6020828403121561539a57615399614f46565b5b5f6153a784828501614f64565b91505092915050565b5f805f805f606086880312156153c9576153c8614f46565b5b5f6153d688828901614f64565b955050602086013567ffffffffffffffff8111156153f7576153f6614f4a565b5b61540388828901614f84565b9450945050604086013567ffffffffffffffff81111561542657615425614f4a565b5b61543288828901615194565b92509250509295509295909350565b5f806040838503121561545757615456614f46565b5b5f61546485828601614f64565b925050602061547585828601614f64565b9150509250929050565b5f81519050919050565b5f82825260208201905092915050565b5f5b838110156154b657808201518184015260208101905061549b565b5f8484015250505050565b5f601f19601f8301169050919050565b5f6154db8261547f565b6154e58185615489565b93506154f5818560208601615499565b6154fe816154c1565b840191505092915050565b5f602083015f8301518482035f86015261552382826154d1565b9150508091505092915050565b5f6020820190508181035f8301526155488184615509565b905092915050565b5f805f806060858703121561556857615567614f46565b5b5f61557587828801614f64565b945050602061558687828801614f64565b935050604085013567ffffffffffffffff8111156155a7576155a6614f4a565b5b6155b387828801615194565b925092505092959194509250565b5f6040820190506155d45f830185615246565b6155e16020830184615246565b9392505050565b5f8115159050919050565b6155fc816155e8565b82525050565b5f6020820190506156155f8301846155f3565b92915050565b5f80fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b615655826154c1565b810181811067ffffffffffffffff821117156156745761567361561f565b5b80604052505050565b5f615686614f3d565b9050615692828261564c565b919050565b5f67ffffffffffffffff8211156156b1576156b061561f565b5b6156ba826154c1565b9050602081019050919050565b828183375f83830152505050565b5f6156e76156e284615697565b61567d565b9050828152602081018484840111156157035761570261561b565b5b61570e8482856156c7565b509392505050565b5f82601f83011261572a57615729614f78565b5b813561573a8482602086016156d5565b91505092915050565b5f806040838503121561575957615758614f46565b5b5f61576685828601615180565b925050602083013567ffffffffffffffff81111561578757615786614f4a565b5b61579385828601615716565b9150509250929050565b5f67ffffffffffffffff82169050919050565b6157b98161579d565b82525050565b5f8160030b9050919050565b6157d4816157bf565b82525050565b5f6040820190506157ed5f8301856157b0565b6157fa60208301846157cb565b9392505050565b5f819050919050565b5f61582461581f61581a8461513a565b615801565b61513a565b9050919050565b5f6158358261580a565b9050919050565b5f6158468261582b565b9050919050565b6158568161583c565b82525050565b5f60208201905061586f5f83018461584d565b92915050565b5f806040838503121561588b5761588a614f46565b5b5f61589885828601614f64565b92505060206158a985828601615180565b9150509250929050565b5f81519050919050565b5f82825260208201905092915050565b5f819050602082019050919050565b5f6158e7838361505f565b60208301905092915050565b5f602082019050919050565b5f615909826158b3565b61591381856158bd565b935061591e836158cd565b805f5b8381101561594e57815161593588826158dc565b9750615940836158f3565b925050600181019050615921565b5085935050505092915050565b5f6020820190508181035f83015261597381846158ff565b905092915050565b5f805f6040848603121561599257615991614f46565b5b5f61599f86828701614f64565b935050602084013567ffffffffffffffff8111156159c0576159bf614f4a565b5b6159cc86828701615194565b92509250509250925092565b5f6020820190506159eb5f8301846157b0565b92915050565b5f81519050919050565b5f82825260208201905092915050565b5f615a15826159f1565b615a1f81856159fb565b9350615a2f818560208601615499565b615a38816154c1565b840191505092915050565b5f6020820190508181035f830152615a5b8184615a0b565b905092915050565b5f60208284031215615a7857615a77614f46565b5b5f615a8584828501615180565b91505092915050565b5f8083601f840112615aa357615aa2614f78565b5b8235905067ffffffffffffffff811115615ac057615abf614f7c565b5b602083019150836020820283011115615adc57615adb614f80565b5b9250929050565b5f805f60408486031215615afa57615af9614f46565b5b5f615b0786828701614f64565b935050602084013567ffffffffffffffff811115615b2857615b27614f4a565b5b615b3486828701615a8e565b92509250509250925092565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b5f615b7782614f0c565b9150615b8283614f0c565b9250828203905081811115615b9a57615b99615b40565b5b92915050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b7f4578747261206461746120746f6f206c617267650000000000000000000000005f82015250565b5f615c016014836159fb565b9150615c0c82615bcd565b602082019050919050565b5f6020820190508181035f830152615c2e81615bf5565b9050919050565b7f737962696c20666565206e6f74206d65740000000000000000000000000000005f82015250565b5f615c696011836159fb565b9150615c7482615c35565b602082019050919050565b5f6020820190508181035f830152615c9681615c5d565b9050919050565b5f615ca78261579d565b915067ffffffffffffffff8203615cc157615cc0615b40565b5b600182019050919050565b5f82825260208201905092915050565b5f615ce78385615ccc565b9350615cf48385846156c7565b615cfd836154c1565b840190509392505050565b5f606082019050615d1b5f830187614f15565b615d286020830186615246565b8181036040830152615d3b818486615cdc565b905095945050505050565b7f50726f6f6620736574206e6f74206c69766500000000000000000000000000005f82015250565b5f615d7a6012836159fb565b9150615d8582615d46565b602082019050919050565b5f6020820190508181035f830152615da781615d6e565b9050919050565b7f4d75737420616464206174206c65617374206f6e6520726f6f740000000000005f82015250565b5f615de2601a836159fb565b9150615ded82615dae565b602082019050919050565b5f6020820190508181035f830152615e0f81615dd6565b9050919050565b7f4f6e6c7920746865206f776e65722063616e2061646420726f6f7473000000005f82015250565b5f615e4a601c836159fb565b9150615e5582615e16565b602082019050919050565b5f6020820190508181035f830152615e7781615e3e565b9050919050565b5f80fd5b5f80fd5b5f80fd5b5f82356001604003833603038112615ea557615ea4615e7e565b5b80830191505092915050565b5f82356001602003833603038112615ecc57615ecb615e7e565b5b80830191505092915050565b5f615ee282614f0c565b9150615eed83614f0c565b9250828201905080821115615f0557615f04615b40565b5b92915050565b5f82825260208201905092915050565b5f819050919050565b5f80fd5b5f82356001602003833603038112615f4357615f42615f24565b5b82810191505092915050565b5f80fd5b5f80fd5b5f8083356001602003843603038112615f7357615f72615f24565b5b83810192508235915060208301925067ffffffffffffffff821115615f9b57615f9a615f4f565b5b600182023603831315615fb157615fb0615f53565b5b509250929050565b5f615fc48385615489565b9350615fd18385846156c7565b615fda836154c1565b840190509392505050565b5f60208301615ff65f840184615f57565b8583035f870152616008838284615fb9565b925050508091505092915050565b5f6160246020840184614f64565b905092915050565b5f6040830161603d5f840184615f28565b8482035f86015261604e8282615fe5565b91505061605e6020840184616016565b61606b602086018261505f565b508091505092915050565b5f616081838361602c565b905092915050565b5f823560016040038336030381126160a4576160a3615f24565b5b82810191505092915050565b5f602082019050919050565b5f6160c78385615f0b565b9350836020840285016160d984615f1b565b805f5b8781101561611c5784840389526160f38284616089565b6160fd8582616076565b9450616108836160b0565b925060208a019950506001810190506160dc565b50829750879450505050509392505050565b5f6080820190506161415f830189614f15565b61614e6020830188614f15565b81810360408301526161618186886160bc565b90508181036060830152616176818486615cdc565b9050979650505050505050565b7f4f6e6c7920746865206f776e65722063616e207363686564756c652072656d6f5f8201527f76616c206f6620726f6f74730000000000000000000000000000000000000000602082015250565b5f6161dd602c836159fb565b91506161e882616183565b604082019050919050565b5f6020820190508181035f83015261620a816161d1565b9050919050565b7f546f6f206d616e792072656d6f76616c73207761697420666f72206e657874205f8201527f70726f76696e6720706572696f6420746f207363686564756c65000000000000602082015250565b5f61626b603a836159fb565b915061627682616211565b604082019050919050565b5f6020820190508181035f8301526162988161625f565b9050919050565b7f43616e206f6e6c79207363686564756c652072656d6f76616c206f66206578695f8201527f7374696e6720726f6f7473000000000000000000000000000000000000000000602082015250565b5f6162f9602b836159fb565b91506163048261629f565b604082019050919050565b5f6020820190508181035f830152616326816162ed565b9050919050565b5f80fd5b82818337505050565b5f61634583856158bd565b93507f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8311156163785761637761632d565b5b602083029250616389838584616331565b82840190509392505050565b5f6060820190506163a85f830188614f15565b81810360208301526163bb81868861633a565b905081810360408301526163d0818486615cdc565b90509695505050505050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b5f600282049050600182168061642057607f821691505b602082108103616433576164326163dc565b5b50919050565b5f819050919050565b61645361644e82614f0c565b616439565b82525050565b5f6164648284616442565b60208201915081905092915050565b5f81905092915050565b5f6164878261547f565b6164918185616473565b93506164a1818560208601615499565b80840191505092915050565b5f6164b8828461647d565b915081905092915050565b7f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c655f8201527f6400000000000000000000000000000000000000000000000000000000000000602082015250565b5f61651d6021836159fb565b9150616528826164c3565b604082019050919050565b5f6020820190508181035f83015261654a81616511565b9050919050565b5f8151905061655f81614f4e565b92915050565b5f6020828403121561657a57616579614f46565b5b5f61658784828501616551565b91505092915050565b7f6f6e6c7920746865206f776e65722063616e206d6f766520746f206e657874205f8201527f70726f76696e6720706572696f64000000000000000000000000000000000000602082015250565b5f6165ea602e836159fb565b91506165f582616590565b604082019050919050565b5f6020820190508181035f830152616617816165de565b9050919050565b7f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c6561765f8201527f6573206172652061646465640000000000000000000000000000000000000000602082015250565b5f616678602c836159fb565b91506166838261661e565b604082019050919050565b5f6020820190508181035f8301526166a58161666c565b9050919050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603160045260245ffd5b7f6368616c6c656e67652065706f6368206d757374206265206174206c656173745f8201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560208201527f2066757475726500000000000000000000000000000000000000000000000000604082015250565b5f6167596047836159fb565b9150616764826166d9565b606082019050919050565b5f6020820190508181035f8301526167868161674d565b9050919050565b5f6080820190506167a05f830188614f15565b6167ad6020830187614f15565b6167ba6040830186614f15565b81810360608301526167cd818486615cdc565b90509695505050505050565b5f6040820190506167ec5f830185614f15565b6167f96020830184614f15565b9392505050565b5f61680a82614f0c565b915061681583614f0c565b925082820261682381614f0c565b9150828204841483151761683a57616839615b40565b5b5092915050565b5f6040820190506168545f83018561535d565b6168616020830184614f15565b9392505050565b5f80fd5b5f8160070b9050919050565b6168818161686c565b811461688b575f80fd5b50565b5f8151905061689c81616878565b92915050565b6168ab8161579d565b81146168b5575f80fd5b50565b5f815190506168c6816168a2565b92915050565b6168d5816157bf565b81146168df575f80fd5b50565b5f815190506168f0816168cc565b92915050565b5f6080828403121561690b5761690a616868565b5b616915608061567d565b90505f6169248482850161688e565b5f830152506020616937848285016168b8565b602083015250604061694b848285016168e2565b604083015250606061695f84828501616551565b60608301525092915050565b5f608082840312156169805761697f614f46565b5b5f61698d848285016168f6565b91505092915050565b7f6661696c656420746f2076616c69646174653a207072696365206d75737420625f8201527f652067726561746572207468616e203000000000000000000000000000000000602082015250565b5f6169f06030836159fb565b91506169fb82616996565b604082019050919050565b5f6020820190508181035f830152616a1d816169e4565b9050919050565b7f4f6e6c79207468652063757272656e74206f776e65722063616e2070726f706f5f8201527f73652061206e6577206f776e6572000000000000000000000000000000000000602082015250565b5f616a7e602e836159fb565b9150616a8982616a24565b604082019050919050565b5f6020820190508181035f830152616aab81616a72565b9050919050565b7f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e20775f8201527f697468207468652076657279206c617374206c656166206f66206120726f6f74602082015250565b5f616b0c6040836159fb565b9150616b1782616ab2565b604082019050919050565b5f6020820190508181035f830152616b3981616b00565b9050919050565b7f70726f6f6620736574206964206f7574206f6620626f756e64730000000000005f82015250565b5f616b74601a836159fb565b9150616b7f82616b40565b602082019050919050565b5f6020820190508181035f830152616ba181616b68565b9050919050565b7f4f6e6c7920746865206f776e65722063616e2064656c6574652070726f6f66205f8201527f7365747300000000000000000000000000000000000000000000000000000000602082015250565b5f616c026024836159fb565b9150616c0d82616ba8565b604082019050919050565b5f6020820190508181035f830152616c2f81616bf6565b9050919050565b5f606082019050616c495f830187614f15565b616c566020830186614f15565b8181036040830152616c69818486615cdc565b905095945050505050565b7f4f6e6c79207468652070726f706f736564206f776e65722063616e20636c61695f8201527f6d206f776e657273686970000000000000000000000000000000000000000000602082015250565b5f616cce602b836159fb565b9150616cd982616c74565b604082019050919050565b5f6020820190508181035f830152616cfb81616cc2565b9050919050565b7f7072656d61747572652070726f6f6600000000000000000000000000000000005f82015250565b5f616d36600f836159fb565b9150616d4182616d02565b602082019050919050565b5f6020820190508181035f830152616d6381616d2a565b9050919050565b7f656d7074792070726f6f660000000000000000000000000000000000000000005f82015250565b5f616d9e600b836159fb565b9150616da982616d6a565b602082019050919050565b5f6020820190508181035f830152616dcb81616d92565b9050919050565b7f6e6f206368616c6c656e6765207363686564756c6564000000000000000000005f82015250565b5f616e066016836159fb565b9150616e1182616dd2565b602082019050919050565b5f6020820190508181035f830152616e3381616dfa565b9050919050565b5f8160c01b9050919050565b5f616e5082616e3a565b9050919050565b616e68616e638261579d565b616e46565b82525050565b5f616e798286616442565b602082019150616e898285616442565b602082019150616e998284616e57565b600882019150819050949350505050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b5f616ee182614f0c565b9150616eec83614f0c565b925082616efc57616efb616eaa565b5b828206905092915050565b5f82356001604003833603038112616f2257616f21615e7e565b5b80830191505092915050565b5f8083356001602003843603038112616f4a57616f49615e7e565b5b80840192508235915067ffffffffffffffff821115616f6c57616f6b615e82565b5b602083019250602082023603831315616f8857616f87615e86565b5b509250929050565b7f70726f6f6620646964206e6f74207665726966790000000000000000000000005f82015250565b5f616fc46014836159fb565b9150616fcf82616f90565b602082019050919050565b5f6020820190508181035f830152616ff181616fb8565b9050919050565b5f60808201905061700b5f830187614f15565b6170186020830186614f15565b6170256040830185614f15565b6170326060830184614f15565b95945050505050565b5f819050919050565b5f61705e6170596170548461703b565b615801565b61579d565b9050919050565b61706e81617044565b82525050565b5f6020820190506170875f830184617065565b92915050565b7f4c65616620696e646578206f7574206f6620626f756e647300000000000000005f82015250565b5f6170c16018836159fb565b91506170cc8261708d565b602082019050919050565b5f6020820190508181035f8301526170ee816170b5565b9050919050565b5f6170ff82614f0c565b91505f820361711157617110615b40565b5b600182039050919050565b5f61712682614f0c565b915061713183614f0c565b92508261714157617140616eaa565b5b828204905092915050565b7f496e636f72726563742066656520616d6f756e740000000000000000000000005f82015250565b5f6171806014836159fb565b915061718b8261714c565b602082019050919050565b5f6020820190508181035f8301526171ad81617174565b9050919050565b50565b5f6171c25f83616473565b91506171cd826171b4565b5f82019050919050565b5f6171e1826171b7565b9150819050919050565b7f4275726e206661696c65640000000000000000000000000000000000000000005f82015250565b5f61721f600b836159fb565b915061722a826171eb565b602082019050919050565b5f6020820190508181035f83015261724c81617213565b9050919050565b7f53697a65206d7573742062652061206d756c7469706c65206f662033320000005f82015250565b5f617287601d836159fb565b915061729282617253565b602082019050919050565b5f6040820190506172b05f830184614f15565b81810360208301526172c18161727b565b905092915050565b7f53697a65206d7573742062652067726561746572207468616e203000000000005f82015250565b5f6172fd601b836159fb565b9150617308826172c9565b602082019050919050565b5f6040820190506173265f830184614f15565b8181036020830152617337816172f1565b905092915050565b7f526f6f742073697a65206d757374206265206c657373207468616e20325e35305f82015250565b5f6173736020836159fb565b915061737e8261733f565b602082019050919050565b5f60408201905061739c5f830184614f15565b81810360208301526173ad81617367565b905092915050565b5f6173bf82614f0c565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82036173f1576173f0615b40565b5b600182019050919050565b5f808335600160200384360303811261741857617417615e7e565b5b80840192508235915067ffffffffffffffff82111561743a57617439615e82565b5b60208301925060018202360383131561745657617455615e86565b5b509250929050565b5f82905092915050565b5f819050815f5260205f209050919050565b5f6020601f8301049050919050565b5f82821b905092915050565b5f600883026174c47fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82617489565b6174ce8683617489565b95508019841693508086168417925050509392505050565b5f6175006174fb6174f684614f0c565b615801565b614f0c565b9050919050565b5f819050919050565b617519836174e6565b61752d61752582617507565b848454617495565b825550505050565b5f90565b617541617535565b61754c818484617510565b505050565b5b8181101561756f576175645f82617539565b600181019050617552565b5050565b601f8211156175b45761758581617468565b61758e8461747a565b8101602085101561759d578190505b6175b16175a98561747a565b830182617551565b50505b505050565b5f82821c905092915050565b5f6175d45f19846008026175b9565b1980831691505092915050565b5f6175ec83836175c5565b9150826002028217905092915050565b617606838361745e565b67ffffffffffffffff81111561761f5761761e61561f565b5b6176298254616409565b617634828285617573565b5f601f831160018114617661575f841561764f578287013590505b61765985826175e1565b8655506176c0565b601f19841661766f86617468565b5f5b8281101561769657848901358255600182019150602085019450602081019050617671565b868310156176b357848901356176af601f8916826175c5565b8355505b6001600288020188555050505b50505050505050565b6176d48383836175fc565b505050565b5f81015f83016176e981856173fc565b6176f48183866176c9565b505050505050565b61770682826176d9565b5050565b7f6661696c656420746f2076616c69646174653a20657374696d617465642067615f8201527f7320666565206d7573742062652067726561746572207468616e203000000000602082015250565b5f617764603c836159fb565b915061776f8261770a565b604082019050919050565b5f6020820190508181035f83015261779181617758565b9050919050565b7f6661696c656420746f2076616c69646174653a204174746f46494c20707269635f8201527f65206d7573742062652067726561746572207468616e20300000000000000000602082015250565b5f6177f26038836159fb565b91506177fd82617798565b604082019050919050565b5f6020820190508181035f83015261781f816177e6565b9050919050565b7f6661696c656420746f2076616c69646174653a207261772073697a65206d75735f8201527f742062652067726561746572207468616e203000000000000000000000000000602082015250565b5f6178806033836159fb565b915061788b82617826565b604082019050919050565b5f6020820190508181035f8301526178ad81617874565b9050919050565b5f8160011c9050919050565b5f808291508390505b6001851115617909578086048111156178e5576178e4615b40565b5b60018516156178f45780820291505b8081029050617902856178b4565b94506178c9565b94509492505050565b5f8261792157600190506179dc565b8161792e575f90506179dc565b8160018114617944576002811461794e5761797d565b60019150506179dc565b60ff8411156179605761795f615b40565b5b8360020a91508482111561797757617976615b40565b5b506179dc565b5060208310610133831016604e8410600b84101617156179b25782820a9050838111156179ad576179ac615b40565b5b6179dc565b6179bf84848460016178c0565b925090508184048111156179d6576179d5615b40565b5b81810290505b9392505050565b5f63ffffffff82169050919050565b5f6179fc82614f0c565b9150617a07836179e3565b9250617a347fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8484617912565b905092915050565b5f617a46826157bf565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008203617a7857617a77615b40565b5b815f039050919050565b617a8b81615354565b8114617a95575f80fd5b50565b5f81519050617aa681617a82565b92915050565b5f60208284031215617ac157617ac0614f46565b5b5f617ace84828501617a98565b91505092915050565b7f436964206461746120697320746f6f2073686f727400000000000000000000005f82015250565b5f617b0b6015836159fb565b9150617b1682617ad7565b602082019050919050565b5f6020820190508181035f830152617b3881617aff565b9050919050565b5f819050602082019050919050565b5f617b598251615354565b80915050919050565b5f617b6c8261547f565b82617b7684617b3f565b9050617b8181617b4e565b92506020821015617bc157617bbc7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83602003600802617489565b831692505b5050919050565b5f606082019050617bdb5f830186614f15565b617be860208301856157b0565b617bf560408301846157cb565b949350505050565b7f496e7075742065786365656473206d6178696d756d20696e743235362076616c5f8201527f7565000000000000000000000000000000000000000000000000000000000000602082015250565b5f617c576022836159fb565b9150617c6282617bfd565b604082019050919050565b5f6020820190508181035f830152617c8481617c4b565b9050919050565b5f819050919050565b5f617c9e82617c8b565b91507f80000000000000000000000000000000000000000000000000000000000000008203617cd057617ccf615b40565b5b815f03905091905056fea264697066735822122033620393606d94f1c8d591222f06f6214eafe22531657caf6478a039d4cfa51864736f6c63430008170033","sourceMap":"1708:31540:17:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2142:50;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;31969:511;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;13160:1027;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;1806:79;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;15412:981;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;2046:90;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;1935:47;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;2390:114;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;9892:181;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;17418:922;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;10634:196;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;9181:183;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;23615:406;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;25066:2013;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;2510:50;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;10149:216;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;;:::i;:::-;;;;;;;;8404:186;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;22003:452;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;4161:214:2;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;32769:477:17;;;;;;;;;;;;;:::i;:::-;;;;;;;;:::i;:::-;;;;;;;;3708:134:2;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;2198:46:17;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;2250:78;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;9649:185;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;12021:517;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;11506:406;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;3155:101:0;;;;;;;;;;;;;:::i;:::-;;8689:441:17;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;14259:848;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;11220:177;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;2441:144:0;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;7976:96:17;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;10905:200;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;1988:52;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;1819:58:2;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;1891:38:17;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;9418:169;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;12544:430;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;2566:43;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;3405:215:0;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;18696:3301:17;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;8149:148;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;7830:103;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;10371:195;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;7269:192;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;:::i;:::-;;2142:50;2188:4;2142:50;:::o;31969:511::-;32057:24;32183:11;32203:29;32214:10;:17;32225:5;32214:17;;;;;;;;;;;;32203:10;:29::i;:::-;32197:3;:35;;;;:::i;:::-;32183:49;;32242:31;32298:10;;:17;;32276:40;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;32242:74;;32331:9;32326:125;32350:10;;:17;;32346:1;:21;32326:125;;;32400:40;32414:5;32421:10;;32432:1;32421:13;;;;;;;:::i;:::-;;;;;;;;32436:3;32400:13;:40::i;:::-;32388:6;32395:1;32388:9;;;;;;;;:::i;:::-;;;;;;;:52;;;;32369:3;;;;;;;32326:125;;;;32467:6;32460:13;;;;31969:511;;;;;:::o;13160:1027::-;13256:7;2188:4;13283:9;;:16;;:39;;13275:72;;;;;;;;;;;;:::i;:::-;;;;;;;;;13357:16;13376:18;:16;:18::i;:::-;13357:37;;13425:8;13412:9;:21;;13404:51;;;;;;;;;;;;:::i;:::-;;;;;;;;;13465:17;13473:8;13465:7;:17::i;:::-;13508:8;13496:9;:20;13492:139;;;13578:10;13570:28;;:50;13611:8;13599:9;:20;;;;:::i;:::-;13570:50;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;13492:139;13641:13;13657:14;;:16;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;13641:32;;;;13710:1;13683:17;:24;13701:5;13683:24;;;;;;;;;;;:28;;;;2559:1;13721:18;:25;13740:5;13721:25;;;;;;;;;;;:50;;;;13855:10;13832:13;:20;13846:5;13832:20;;;;;;;;;;;;:33;;;;;;;;;;;;;;;;;;13901:12;13875:16;:23;13892:5;13875:23;;;;;;;;;;;;:38;;;;;;;;;;;;;;;;;;2608:1;13923:23;:30;13947:5;13923:30;;;;;;;;;;;:48;;;;14010:1;13986:26;;:12;:26;;;13982:128;;14040:12;14028:41;;;14070:5;14077:10;14089:9;;14028:71;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;13982:128;14147:10;14124:34;;14140:5;14124:34;;;;;;;;;;14175:5;14168:12;;;;13160:1027;;;;;:::o;1806:79::-;1843:42;1806:79;:::o;15412:981::-;15517:7;2188:4;15544:9;;:16;;:39;;15536:72;;;;;;;;;;;;:::i;:::-;;;;;;;;;15626:19;15639:5;15626:12;:19::i;:::-;15618:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;15704:1;15686:8;;:15;;:19;15678:58;;;;;;;;;;;;:::i;:::-;;;;;;;;;15778:10;15754:34;;:13;:20;15768:5;15754:20;;;;;;;;;;;;;;;;;;;;;:34;;;15746:75;;;;;;;;;;;;:::i;:::-;;;;;;;;;15831:18;15852:10;:17;15863:5;15852:17;;;;;;;;;;;;15831:38;;15879:24;15920:8;;:15;;15906:30;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;15879:57;;15953:9;15948:171;15972:8;;:15;;15968:1;:19;15948:171;;;16008:59;16019:5;16026:1;16029:8;;16038:1;16029:11;;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;:16;;;;;;;;:::i;:::-;16047:8;;16056:1;16047:11;;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;:19;;;16008:10;:59::i;:::-;;16107:1;16094:10;:14;;;;:::i;:::-;16081:7;16089:1;16081:10;;;;;;;;:::i;:::-;;;;;;;:27;;;;;15989:3;;;;;;;15948:171;;;;16144:5;16133:26;16151:7;16133:26;;;;;;:::i;:::-;;;;;;;;16170:20;16193:16;:23;16210:5;16193:23;;;;;;;;;;;;;;;;;;;;;16170:46;;16254:1;16230:26;;:12;:26;;;16226:133;;16284:12;16272:36;;;16309:5;16316:10;16328:8;;16338:9;;16272:76;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;16226:133;16376:10;16369:17;;;;;15412:981;;;;;;;:::o;2046:90::-;2094:42;2046:90;:::o;1935:47::-;1975:7;1935:47;:::o;2390:114::-;2438:66;2390:114;;;:::o;9892:181::-;9957:7;9984:19;9997:5;9984:12;:19::i;:::-;9976:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;10043:16;:23;10060:5;10043:23;;;;;;;;;;;;;;;;;;;;;10036:30;;9892:181;;;:::o;17418:922::-;2188:4;17538:9;;:16;;:39;;17530:72;;;;;;;;;;;;:::i;:::-;;;;;;;;;17620:19;17633:5;17620:12;:19::i;:::-;17612:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;17704:10;17680:34;;:13;:20;17694:5;17680:20;;;;;;;;;;;;;;;;;;;;;:34;;;17672:91;;;;;;;;;;;;:::i;:::-;;;;;;;;;2036:4;17798:17;:24;17816:5;17798:24;;;;;;;;;;;:31;;;;17781:7;;:14;;:48;;;;:::i;:::-;:73;;17773:144;;;;;;;;;;;;:::i;:::-;;;;;;;;;17933:9;17928:210;17952:7;;:14;;17948:1;:18;17928:210;;;18007:10;:17;18018:5;18007:17;;;;;;;;;;;;17994:7;;18002:1;17994:10;;;;;;;:::i;:::-;;;;;;;;:30;17986:86;;;;;;;;;;;;:::i;:::-;;;;;;;;;18086:17;:24;18104:5;18086:24;;;;;;;;;;;18116:7;;18124:1;18116:10;;;;;;;:::i;:::-;;;;;;;;18086:41;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;17968:3;;;;;;;17928:210;;;;18148:20;18171:16;:23;18188:5;18171:23;;;;;;;;;;;;;;;;;;;;;18148:46;;18232:1;18208:26;;:12;:26;;;18204:130;;18262:12;18250:46;;;18297:5;18304:7;;18313:9;;18250:73;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;18204:130;17520:820;17418:922;;;;;:::o;10634:196::-;10706:15;;:::i;:::-;10741:19;10754:5;10741:12;:19::i;:::-;10733:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;10800:8;:15;10809:5;10800:15;;;;;;;;;;;:23;10816:6;10800:23;;;;;;;;;;;10793:30;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;10634:196;;;;:::o;9181:183::-;9247:7;9274:19;9287:5;9274:12;:19::i;:::-;9266:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;9333:17;:24;9351:5;9333:24;;;;;;;;;;;;9326:31;;9181:183;;;:::o;23615:406::-;23674:7;23725:12;23739:19;2094:42;23762:32;;23812:5;23795:23;;;;;;;;:::i;:::-;;;;;;;;;;;;;23762:57;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;23724:95;;;;23882:7;23874:53;;;;;;;;;;;;:::i;:::-;;;;;;;;;23996:6;23985:29;;;;;;;;;;;;:::i;:::-;23978:36;;;;23615:406;;;:::o;25066:2013::-;2188:4;25183:9;;:16;;:39;;25175:72;;;;;;;;;;;;:::i;:::-;;;;;;;;;25279:13;:20;25293:5;25279:20;;;;;;;;;;;;;;;;;;;;;25265:34;;:10;:34;;;25257:93;;;;;;;;;;;;:::i;:::-;;;;;;;;;25395:1;25368:17;:24;25386:5;25368:24;;;;;;;;;;;;:28;25360:85;;;;;;;;;;;;:::i;:::-;;;;;;;;;2608:1;25468:23;:30;25492:5;25468:30;;;;;;;;;;;;:49;25464:125;;25566:12;25533:23;:30;25557:5;25533:30;;;;;;;;;;;:45;;;;25464:125;25648:26;25677:17;:24;25695:5;25677:24;;;;;;;;;;;25648:53;;25711:34;25762:8;:15;;;;25748:30;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;25711:67;;25794:9;25789:160;25813:17;:24;25809:1;:28;25789:160;;;25881:8;25908:1;25890:8;:15;;;;:19;;;;:::i;:::-;25881:29;;;;;;;;:::i;:::-;;;;;;;;;;25858:17;25876:1;25858:20;;;;;;;;:::i;:::-;;;;;;;:52;;;;;25924:8;:14;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;25839:3;;;;;;;25789:160;;;;25959:37;25971:5;25978:17;25959:11;:37::i;:::-;26024:5;26011:38;26031:17;26011:38;;;;;;:::i;:::-;;;;;;;;26138:17;:24;26156:5;26138:24;;;;;;;;;;;;26114:14;:21;26129:5;26114:21;;;;;;;;;;;:48;;;;26208:17;;26193:12;:32;;;;:::i;:::-;26176:14;:49;26172:161;;;26241:81;;;;;;;;;;:::i;:::-;;;;;;;;26172:161;26370:14;26342:18;:25;26361:5;26342:25;;;;;;;;;;;:42;;;;26576:1;26548:17;:24;26566:5;26548:24;;;;;;;;;;;;:29;26544:211;;26612:5;26598:20;;;;;;;;;;2608:1;26632:23;:30;26656:5;26632:30;;;;;;;;;;;:48;;;;2559:1;26694:18;:25;26713:5;26694:25;;;;;;;;;;;:50;;;;26544:211;26765:20;26788:16;:23;26805:5;26788:23;;;;;;;;;;;;;;;;;;;;;26765:46;;26849:1;26825:26;;:12;:26;;;26821:171;;26879:12;26867:43;;;26911:5;26918:18;:25;26937:5;26918:25;;;;;;;;;;;;26945:17;:24;26963:5;26945:24;;;;;;;;;;;;26971:9;;26867:114;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;26821:171;27024:5;27006:66;27031:14;27047:17;:24;27065:5;27047:24;;;;;;;;;;;;27006:66;;;;;;;:::i;:::-;;;;;;;;25165:1914;;;25066:2013;;;;:::o;2510:50::-;2559:1;2510:50;:::o;10149:216::-;10211:7;10220;10247:19;10260:5;10247:12;:19::i;:::-;10239:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;10307:13;:20;10321:5;10307:20;;;;;;;;;;;;;;;;;;;;;10329:21;:28;10351:5;10329:28;;;;;;;;;;;;;;;;;;;;;10299:59;;;;10149:216;;;:::o;8404:186::-;8474:4;8497:19;8510:5;8497:12;:19::i;:::-;:49;;;;;8529:10;:17;8540:5;8529:17;;;;;;;;;;;;8520:6;:26;8497:49;:86;;;;;8582:1;8550:14;:21;8565:5;8550:21;;;;;;;;;;;:29;8572:6;8550:29;;;;;;;;;;;;:33;8497:86;8490:93;;8404:186;;;;:::o;22003:452::-;22091:7;22110:15;22133:14;:21;22148:5;22133:21;;;;;;;;;;;;22128:2;:26;;;;:::i;:::-;22110:44;;22165:18;22185:21;22210:16;:14;:16::i;:::-;22164:62;;;;22244:204;22289:15;22318:11;22343:15;22372:7;22408:23;:30;22432:5;22408:30;;;;;;;;;;;;22393:12;:45;;;;:::i;:::-;22244:31;:204::i;:::-;22237:211;;;;;22003:452;;;;:::o;4161:214:2:-;2655:13;:11;:13::i;:::-;4276:36:::1;4294:17;4276;:36::i;:::-;4322:46;4344:17;4363:4;4322:21;:46::i;:::-;4161:214:::0;;:::o;32769:477:17:-;32816:6;32824:5;32890:34;2285:42;32927:24;;;2438:66;32965:21;;2239:5;32927:97;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;32890:134;;33060:1;33042:9;:15;;;:19;;;33034:80;;;;;;;;;;;;:::i;:::-;;;;;;;;;33206:9;:15;;;33224:9;:14;;;33191:48;;;;;32769:477;;:::o;3708:134:2:-;3777:7;2926:20;:18;:20::i;:::-;811:66:6::1;3803:32:2;;3796:39;;3708:134:::0;:::o;2198:46:17:-;2239:5;2198:46;:::o;2250:78::-;2285:42;2250:78;:::o;9649:185::-;9716:7;9743:19;9756:5;9743:12;:19::i;:::-;9735:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;9802:18;:25;9821:5;9802:25;;;;;;;;;;;;9795:32;;9649:185;;;:::o;12021:517::-;12109:19;12122:5;12109:12;:19::i;:::-;12101:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;12161:13;12177;:20;12191:5;12177:20;;;;;;;;;;;;;;;;;;;;;12161:36;;12224:10;12215:19;;:5;:19;;;12207:78;;;;;;;;;;;;:::i;:::-;;;;;;;;;12308:8;12299:17;;:5;:17;;;12295:237;;12423:21;:28;12445:5;12423:28;;;;;;;;;;;;12416:35;;;;;;;;;;;12295:237;;;12513:8;12482:21;:28;12504:5;12482:28;;;;;;;;;;;;:39;;;;;;;;;;;;;;;;;;12295:237;12091:447;12021:517;;:::o;11506:406::-;11572:16;11608:19;11621:5;11608:12;:19::i;:::-;11600:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;11660:26;11689:17;:24;11707:5;11689:24;;;;;;;;;;;11660:53;;11723:23;11763:8;:15;;;;11749:30;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;11723:56;;11794:9;11789:94;11813:8;:15;;;;11809:1;:19;11789:94;;;11861:8;11870:1;11861:11;;;;;;;;:::i;:::-;;;;;;;;;;11849:6;11856:1;11849:9;;;;;;;;:::i;:::-;;;;;;;:23;;;;;11830:3;;;;;;;11789:94;;;;11899:6;11892:13;;;;11506:406;;;:::o;3155:101:0:-;2334:13;:11;:13::i;:::-;3219:30:::1;3246:1;3219:18;:30::i;:::-;3155:101::o:0;8689:441:17:-;8767:4;8783:11;8803:29;8814:10;:17;8825:5;8814:17;;;;;;;;;;;;8803:10;:29::i;:::-;8797:3;:35;;;;:::i;:::-;8783:49;;8842:26;8871:50;8885:5;8914:1;8892:14;:21;8907:5;8892:21;;;;;;;;;;;;:23;;;;:::i;:::-;8917:3;8871:13;:50::i;:::-;8842:79;;8989:1;8953:14;:21;8968:5;8953:21;;;;;;;;;;;:33;8975:3;:10;;;8953:33;;;;;;;;;;;;:37;;;;:::i;:::-;8939:3;:10;;;:51;8931:128;;;;;;;;;;;;:::i;:::-;;;;;;;;;9076:23;9085:5;9092:6;9076:8;:23::i;:::-;:47;;;;;9113:3;:10;;;9103:6;:20;;9076:47;9069:54;;;;8689:441;;;;:::o;14259:848::-;2188:4;14349:9;;:16;;:39;;14341:72;;;;;;;;;;;;:::i;:::-;;;;;;;;;14436:14;;;;;;;;;;;14427:23;;:5;:23;14423:90;;14466:36;;;;;;;;;;:::i;:::-;;;;;;;;14423:90;14555:10;14531:34;;:13;:20;14545:5;14531:20;;;;;;;;;;;;;;;;;;;;;:34;;;14523:83;;;;;;;;;;;;:::i;:::-;;;;;;;;;14616:24;14643:17;:24;14661:5;14643:24;;;;;;;;;;;;14616:51;;14704:1;14677:17;:24;14695:5;14677:24;;;;;;;;;;;:28;;;;14746:1;14715:13;:20;14729:5;14715:20;;;;;;;;;;;;:33;;;;;;;;;;;;;;;;;;14786:1;14758:18;:25;14777:5;14758:25;;;;;;;;;;;:29;;;;2608:1;14797:23;:30;14821:5;14797:30;;;;;;;;;;;:48;;;;14856:20;14879:16;:23;14896:5;14879:23;;;;;;;;;;;;;;;;;;;;;14856:46;;14940:1;14916:26;;:12;:26;;;14912:134;;14970:12;14958:41;;;15000:5;15007:16;15025:9;;14958:77;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;14912:134;15076:5;15060:40;15083:16;15060:40;;;;;;:::i;:::-;;;;;;;;14331:776;;14259:848;;;:::o;11220:177::-;11283:7;11310:19;11323:5;11310:12;:19::i;:::-;11302:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;11369:14;:21;11384:5;11369:21;;;;;;;;;;;;11362:28;;11220:177;;;:::o;2441:144:0:-;2487:7;2506:24;2533:20;:18;:20::i;:::-;2506:47;;2570:1;:8;;;;;;;;;;;;2563:15;;;2441:144;:::o;7976:96:17:-;8026:6;8051:14;;;;;;;;;;;8044:21;;7976:96;:::o;10905:200::-;10983:7;11010:19;11023:5;11010:12;:19::i;:::-;11002:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;11069:14;:21;11084:5;11069:21;;;;;;;;;;;:29;11091:6;11069:29;;;;;;;;;;;;11062:36;;10905:200;;;;:::o;1988:52::-;2036:4;1988:52;:::o;1819:58:2:-;;;;;;;;;;;;;;;;;;;:::o;1891:38:17:-;1927:2;1891:38;:::o;9418:169::-;9477:7;9504:19;9517:5;9504:12;:19::i;:::-;9496:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;9563:10;:17;9574:5;9563:17;;;;;;;;;;;;9556:24;;9418:169;;;:::o;12544:430::-;12616:19;12629:5;12616:12;:19::i;:::-;12608:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;12708:10;12676:42;;:21;:28;12698:5;12676:28;;;;;;;;;;;;;;;;;;;;;:42;;;12668:98;;;;;;;;;;;;:::i;:::-;;;;;;;;;12776:16;12795:13;:20;12809:5;12795:20;;;;;;;;;;;;;;;;;;;;;12776:39;;12848:10;12825:13;:20;12839:5;12825:20;;;;;;;;;;;;:33;;;;;;;;;;;;;;;;;;12875:21;:28;12897:5;12875:28;;;;;;;;;;;;12868:35;;;;;;;;;;;12956:10;12918:49;;12946:8;12918:49;;12939:5;12918:49;;;;;;;;;;12598:376;12544:430;:::o;2566:43::-;2608:1;2566:43;:::o;3405:215:0:-;2334:13;:11;:13::i;:::-;3509:1:::1;3489:22;;:8;:22;;::::0;3485:91:::1;;3562:1;3534:31;;;;;;;;;;;:::i;:::-;;;;;;;;3485:91;3585:28;3604:8;3585:18;:28::i;:::-;3405:215:::0;:::o;18696:3301:17:-;18785:18;18806:9;18785:30;;18847:13;:20;18861:5;18847:20;;;;;;;;;;;;;;;;;;;;;18833:34;;:10;:34;;;18825:93;;;;;;;;;;;;:::i;:::-;;;;;;;;;18928:22;18953:18;:25;18972:5;18953:25;;;;;;;;;;;;18928:50;;19012:14;18996:12;:30;;18988:58;;;;;;;;;;;;:::i;:::-;;;;;;;;;19080:1;19064:6;;:13;;:17;19056:41;;;;;;;;;;;;:::i;:::-;;;;;;;;;2559:1;19115:14;:40;19107:75;;;;;;;;;;;;:::i;:::-;;;;;;;;;19192:35;19252:6;;:13;;19230:36;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;19192:74;;19285:12;19300:24;19318:5;19300:17;:24::i;:::-;19285:39;;19334:17;19354:14;:21;19369:5;19354:21;;;;;;;;;;;;19334:41;;19385:18;19412:29;19423:10;:17;19434:5;19423:17;;;;;;;;;;;;19412:10;:29::i;:::-;19406:3;:35;;;;:::i;:::-;19385:56;;19456:8;19451:1695;19474:6;;:13;;19470:1;:17;;;19451:1695;;;20584:20;20624:4;20630:5;20637:1;20607:32;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;20584:55;;20653:20;20706:9;20694:7;20684:18;;;;;;20676:27;;:39;;;;:::i;:::-;20653:62;;20840:46;20854:5;20861:12;20875:10;20840:13;:46::i;:::-;20824:10;20835:1;20824:13;;;;;;;;;;:::i;:::-;;;;;;;:62;;;;20900:16;20919:59;20938:39;20949:5;20956:10;20967:1;20956:13;;;;;;;;;;:::i;:::-;;;;;;;;:20;;;20938:10;:39::i;:::-;20919:18;:59::i;:::-;20900:78;;20992:7;21002:84;21022:6;;21029:1;21022:9;;;;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;:15;;;;;;;;:::i;:::-;21002:84;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21039:8;21049:6;;21056:1;21049:9;;;;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;:14;;;21065:10;21076:1;21065:13;;;;;;;;;;:::i;:::-;;;;;;;;:20;;;21002:19;:84::i;:::-;20992:94;;21108:2;21100:35;;;;;;;;;;;;:::i;:::-;;;;;;;;;19494:1652;;;;19489:3;;;;;:::i;:::-;;;;19451:1695;;;;21526:15;21611:4;21605:2;21573:29;21595:6;;21573:21;:29::i;:::-;:34;;;;:::i;:::-;21572:43;;;;:::i;:::-;21558:9;21545:10;:22;;;;:::i;:::-;21544:72;;;;:::i;:::-;21526:90;;21626:40;21651:5;21658:7;21626:24;:40::i;:::-;21677:20;21700:16;:23;21717:5;21700:23;;;;;;;;;;;;;;;;;;;;;21677:46;;21761:1;21737:26;;:12;:26;;;21733:153;;21791:12;21779:42;;;21822:5;21829:17;:24;21847:5;21829:24;;;;;;;;;;;;21855:4;21861:6;;:13;;21779:96;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21733:153;21928:12;21895:23;:30;21919:5;21895:30;;;;;;;;;;;:45;;;;21972:5;21955:35;21979:10;21955:35;;;;;;:::i;:::-;;;;;;;;18775:3222;;;;;;;;18696:3301;;;:::o;8149:148::-;8207:4;8238:14;;;;;;;;;;;8230:22;;:5;:22;:60;;;;;8288:1;8256:34;;:13;:20;8270:5;8256:20;;;;;;;;;;;;;;;;;;;;;:34;;;;8230:60;8223:67;;8149:148;;;:::o;7830:103::-;7883:7;7909:17;;7902:24;;7830:103;:::o;10371:195::-;10443:7;10470:19;10483:5;10470:12;:19::i;:::-;10462:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;10529:23;:30;10553:5;10529:30;;;;;;;;;;;;10522:37;;10371:195;;;:::o;7269:192::-;4158:30:1;4191:26;:24;:26::i;:::-;4158:59;;4279:19;4302:1;:15;;;;;;;;;;;;4301:16;4279:38;;4327:18;4348:1;:14;;;;;;;;;;;;4327:35;;4706:17;4741:1;4726:11;:16;;;:34;;;;;4746:14;4726:34;4706:54;;4770:17;4805:1;4790:11;:16;;;:50;;;;;4839:1;4818:4;4810:25;;;:30;4790:50;4770:70;;4856:12;4855:13;:30;;;;;4873:12;4872:13;4855:30;4851:91;;;4908:23;;;;;;;;;;;;;;4851:91;4968:1;4951;:14;;;:18;;;;;;;;;;;;;;;;;;4983:14;4979:67;;;5031:4;5013:1;:15;;;:22;;;;;;;;;;;;;;;;;;4979:67;7346:26:17::1;7361:10;7346:14;:26::i;:::-;7382:24;:22;:24::i;:::-;7436:18;7416:17;:38;;;;5070:14:1::0;5066:101;;;5118:5;5100:1;:15;;;:23;;;;;;;;;;;;;;;;;;5142:14;5154:1;5142:14;;;;;;:::i;:::-;;;;;;;;5066:101;4092:1081;;;;;7269:192:17;:::o;189:563:14:-;236:7;255:9;267:3;255:15;;280:9;309:3;304:1;:8;;300:12;;323:1;318;:6;314:32;;333:3;328:8;;;;;:::i;:::-;;;342:1;338:5;;314:32;364:2;359:1;:7;;355:11;;378:1;373;:6;369:32;;388:2;383:7;;;;;:::i;:::-;;;397:1;393:5;;369:32;419:2;414:1;:7;;410:11;;433:1;428;:6;424:32;;443:2;438:7;;;;;:::i;:::-;;;452:1;448:5;;424:32;474:2;469:1;:7;;465:11;;488:1;483;:6;479:32;;498:2;493:7;;;;;:::i;:::-;;;507:1;503:5;;479:32;529:1;524;:6;;520:10;;543:1;538;:6;534:32;;553:1;548:6;;;;;:::i;:::-;;;562:1;558:5;;534:32;584:1;579;:6;;575:10;;598:1;593;:6;589:32;;608:1;603:6;;;;;:::i;:::-;;;617:1;613:5;;589:32;639:1;634;:6;;630:10;;653:1;648;:6;644:32;;663:1;658:6;;;;;:::i;:::-;;;672:1;668:5;;644:32;694:1;689;:6;;685:10;;708:1;703;:6;699:24;;722:1;718;:5;;;;:::i;:::-;711:12;;;;;;699:24;744:1;740;:5;;;;:::i;:::-;733:12;;;;189:563;;;;:::o;30599:1307:17:-;30692:22;;:::i;:::-;30746:17;:24;30764:5;30746:24;;;;;;;;;;;;30734:9;:36;30726:73;;;;;;;;;;;;:::i;:::-;;;;;;;;;30809:17;30842:1;30835:3;30830:1;:8;;30829:14;;;;:::i;:::-;30809:34;;30853:11;30973:17;31005:9;31017:3;31005:15;;31000:616;31026:1;31022;:5;31000:616;;;31169:10;:17;31180:5;31169:17;;;;;;;;;;;;31156:9;:30;31152:120;;31229:1;31225;:5;;;;:::i;:::-;31219:1;:12;;31206:25;;;;;:::i;:::-;;;31249:8;;31152:120;31304:13;:20;31318:5;31304:20;;;;;;;;;;;:31;31325:9;31304:31;;;;;;;;;;;;31298:3;:37;;;;:::i;:::-;31286:49;;31390:9;31377;:22;31373:233;;31426:13;:20;31440:5;31426:20;;;;;;;;;;;:31;31447:9;31426:31;;;;;;;;;;;;31419:38;;;;;:::i;:::-;;;31498:1;31494;:5;;;;:::i;:::-;31488:1;:12;;31475:25;;;;;:::i;:::-;;;31373:233;;;31589:1;31585;:5;;;;:::i;:::-;31579:1;:12;;31566:25;;;;;:::i;:::-;;;31373:233;31000:616;31029:3;;;;;:::i;:::-;;;;31000:616;;;;31643:13;:20;31657:5;31643:20;;;;;;;;;;;:31;31664:9;31643:31;;;;;;;;;;;;31637:3;:37;;;;:::i;:::-;31625:49;;31701:9;31688;:22;31684:141;;31761:53;;;;;;;;31789:1;31777:9;:13;;;;:::i;:::-;31761:53;;;;31804:9;31792;:21;;;;:::i;:::-;31761:53;;;31754:60;;;;;;;31684:141;31856:43;;;;;;;;31872:9;31856:43;;;;31895:3;31883:9;:15;;;;:::i;:::-;31856:43;;;31849:50;;;;;30599:1307;;;;;;:::o;3623:85:16:-;3666:7;354:2;228:1;270:4;:15;;;;:::i;:::-;336:20;;;;:::i;:::-;3685:16;;3623:85;:::o;7557:215:17:-;7630:6;7617:9;:19;;7609:52;;;;;;;;;;;;:::i;:::-;;;;;;;;;7672:12;1843:42;7690:15;;7713:6;7690:34;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;7671:53;;;7742:7;7734:31;;;;;;;;;;;;:::i;:::-;;;;;;;;;7599:173;7557:215;:::o;16449:793::-;16560:7;16606:1;1927:2;16583:7;:19;;;;:::i;:::-;:24;16579:116;;16643:7;16630:54;;;;;;;;;;;:::i;:::-;;;;;;;;16579:116;16719:1;16708:7;:12;16704:102;;16756:7;16743:52;;;;;;;;;;;:::i;:::-;;;;;;;;16704:102;1975:7;16819;:23;16815:118;;;16878:7;16865:57;;;;;;;;;;;:::i;:::-;;;;;;;;16815:118;16943:17;1927:2;16963:7;:19;;;;:::i;:::-;16943:39;;16992:14;17009:10;:17;17020:5;17009:17;;;;;;;;;;;;:19;;;;;;;;;:::i;:::-;;;;;16992:36;;17038;17049:5;17056:9;17067:6;17038:10;:36::i;:::-;17110:4;17084:8;:15;17093:5;17084:15;;;;;;;;;;;:23;17100:6;17084:23;;;;;;;;;;;:30;;;;;;:::i;:::-;;;;17156:9;17124:14;:21;17139:5;17124:21;;;;;;;;;;;:29;17146:6;17124:29;;;;;;;;;;;:41;;;;17203:9;17175:17;:24;17193:5;17175:24;;;;;;;;;;;;:37;;;;;;;:::i;:::-;;;;;;;;17229:6;17222:13;;;;16449:793;;;;;;:::o;27132:342::-;27221:19;27234:5;27221:12;:19::i;:::-;27213:50;;;;;;;;;;;;:::i;:::-;;;;;;;;;27273:18;27310:9;27305:115;27329:7;:14;27325:1;:18;27305:115;;;27377:32;27391:5;27398:7;27406:1;27398:10;;;;;;;;:::i;:::-;;;;;;;;27377:13;:32::i;:::-;27363:46;;;;;:::i;:::-;;;27345:3;;;;;;;27305:115;;;;27457:10;27429:17;:24;27447:5;27429:24;;;;;;;;;;;;:38;;;;;;;:::i;:::-;;;;;;;;27203:271;27132:342;;:::o;1662:1834:16:-;1885:7;1938:1;1920:15;:19;:41;;;;1960:1;1943:13;:18;1920:41;1912:114;;;;;;;;;;;;:::i;:::-;;;;;;;;;2058:1;2044:11;:15;;;2036:84;;;;;;;;;;;;:::i;:::-;;;;;;;;;2148:1;2138:7;:11;2130:75;;;;;;;;;;;;:::i;:::-;;;;;;;;;2276:29;2338:1;2319:15;:20;;;2315:452;;2522:15;2509:2;:29;;;;:::i;:::-;2494:11;2460:45;;1018:5;903:7;2460:31;;;;:::i;:::-;:45;;;;:::i;:::-;:79;;;;:::i;:::-;228:1;270:4;:15;;;;:::i;:::-;496:1;2380:58;;;;:::i;:::-;2379:161;;;;:::i;:::-;2355:185;;2315:452;;;2744:11;2710:45;;1018:5;903:7;2710:31;;;;:::i;:::-;:45;;;;:::i;:::-;2672:15;2671:16;;;:::i;:::-;2658:2;:30;;;;:::i;:::-;228:1;270:4;:15;;;;:::i;:::-;496:1;2596:58;;;;:::i;:::-;:93;;;;:::i;:::-;2595:161;;;;:::i;:::-;2571:185;;2315:452;2834:30;2906:7;2891:12;2867:21;:36;;;;:::i;:::-;:46;;;;:::i;:::-;2834:79;;2956:21;3036:3;792:1;2981:22;:51;;;;:::i;:::-;2980:59;;;;:::i;:::-;2956:83;;3049:20;3127:3;682:1;3073:22;:50;;;;:::i;:::-;3072:58;;;;:::i;:::-;3049:81;;3164:13;3145:15;:32;3141:349;;3200:1;3193:8;;;;;;;;3141:349;3289:12;3270:15;:31;3266:224;;3340:15;3324:13;:31;;;;:::i;:::-;3317:38;;;;;;;;3266:224;3476:3;574:1;3427:22;:45;;;;:::i;:::-;3426:53;;;;:::i;:::-;3419:60;;;;;;1662:1834;;;;;;;;:::o;4603:312:2:-;4692:6;4675:23;;4683:4;4675:23;;;:120;;;;4789:6;4753:42;;:32;:30;:32::i;:::-;:42;;;;4675:120;4658:251;;;4869:29;;;;;;;;;;;;;;4658:251;4603:312::o;7467:84:17:-;2334:13:0;:11;:13::i;:::-;7467:84:17;:::o;6057:538:2:-;6174:17;6156:50;;;:52;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;6152:437;;6560:17;6518:60;;;;;;;;;;;:::i;:::-;;;;;;;;6152:437;811:66:6;6258:32:2;;6250:4;:40;6246:120;;6346:4;6317:34;;;;;;;;;;;:::i;:::-;;;;;;;;6246:120;6379:54;6409:17;6428:4;6379:29;:54::i;:::-;6209:235;6057:538;;:::o;5032:213::-;5115:6;5098:23;;5106:4;5098:23;;;5094:145;;5199:29;;;;;;;;;;;;;;5094:145;5032:213::o;2658:162:0:-;2728:12;:10;:12::i;:::-;2717:23;;:7;:5;:7::i;:::-;:23;;;2713:101;;2790:12;:10;:12::i;:::-;2763:40;;;;;;;;;;;:::i;:::-;;;;;;;;2713:101;2658:162::o;3774:248::-;3847:24;3874:20;:18;:20::i;:::-;3847:47;;3904:16;3923:1;:8;;;;;;;;;;;;3904:27;;3952:8;3941:1;:8;;;:19;;;;;;;;;;;;;;;;;;4006:8;3975:40;;3996:8;3975:40;;;;;;;;;;;;3837:185;;3774:248;:::o;1192:159::-;1244:24;1313:22;1303:32;;1192:159;:::o;24028:138:17:-;24093:7;24119:40;24133:18;:25;24152:5;24133:25;;;;;;;;;;;;24119:13;:40::i;:::-;24112:47;;24028:138;;;:::o;274:339:15:-;336:7;382:2;363:3;:8;;;:15;:21;;355:55;;;;;;;;;;;;:::i;:::-;;;;;;;;;420:22;455:2;445:13;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;420:38;;473:6;468:104;489:2;485:1;:6;468:104;;;527:3;:8;;;559:1;554:2;536:3;:8;;;:15;:20;;;;:::i;:::-;:24;;;;:::i;:::-;527:34;;;;;;;;:::i;:::-;;;;;;;;;;512:9;522:1;512:12;;;;;;;;:::i;:::-;;;;;:49;;;;;;;;;;;493:3;;;;;;;468:104;;;;596:9;588:18;;;:::i;:::-;581:25;;;274:339;;;:::o;951:188:18:-;1060:4;1128;1083:41;1102:5;1109:4;1115:8;1083:18;:41::i;:::-;:49;1076:56;;951:188;;;;;;:::o;23225:384:17:-;23304:7;23323:20;23346:1;23323:24;;23362:9;23357:217;23381:6;;:13;;23377:1;:17;23357:217;;;23560:2;23535:6;;23542:1;23535:9;;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;:15;;;;;;;;:::i;:::-;:22;;:27;;;;:::i;:::-;23529:2;:34;;;;:::i;:::-;23513:50;;;;;:::i;:::-;;;23396:3;;;;;;;23357:217;;;;23590:12;23583:19;;;23225:384;;;;:::o;22461:758::-;22546:23;22582:13;22572:7;:23;;;;:::i;:::-;22546:49;;22605:15;22628:14;:21;22643:5;22628:21;;;;;;;;;;;;22623:2;:26;;;;:::i;:::-;22605:44;;22660:18;22680:21;22705:16;:14;:16::i;:::-;22659:62;;;;22732:16;22751:204;22796:15;22825:11;22850:15;22879:7;22915:23;:30;22939:5;22915:30;;;;;;;;;;;;22900:12;:45;;;;:::i;:::-;22751:31;:204::i;:::-;22732:223;;22973:17;22981:8;22973:7;:17::i;:::-;23016:8;23004:9;:20;23000:139;;;23086:10;23078:28;;:50;23119:8;23107:9;:20;;;;:::i;:::-;23078:50;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;23000:139;23166:5;23153:59;23173:8;23183:11;23196:15;23153:59;;;;;;;;:::i;:::-;;;;;;;;22536:683;;;;;22461:758;;:::o;8737:170:1:-;8795:30;8870:21;8860:31;;8737:170;:::o;1847:127:0:-;6931:20:1;:18;:20::i;:::-;1929:38:0::1;1954:12;1929:24;:38::i;:::-;1847:127:::0;:::o;2970:67:2:-;6931:20:1;:18;:20::i;:::-;2970:67:2:o;29480:420:17:-;29565:13;29581:6;29565:22;;29597:9;29609:22;29625:5;29609:15;:22::i;:::-;29597:34;;29642:11;29656:5;29642:19;;29726:9;29721:129;29745:1;29741;:5;29721:129;;;29767:9;29793:1;29788;:6;;29779:5;:16;;;;:::i;:::-;29767:28;;29816:13;:20;29830:5;29816:20;;;;;;;;;;;:23;29837:1;29816:23;;;;;;;;;;;;29809:30;;;;;:::i;:::-;;;29753:97;29748:3;;;;;;;29721:129;;;;29890:3;29859:13;:20;29873:5;29859:20;;;;;;;;;;;:28;29880:6;29859:28;;;;;;;;;;;:34;;;;29555:345;;;29480:420;;;:::o;27648:296::-;27720:7;27739:13;27755:14;:21;27770:5;27755:21;;;;;;;;;;;:29;27777:6;27755:29;;;;;;;;;;;;27739:45;;27794:35;27808:5;27815:6;27823:5;27794:13;:35::i;:::-;27846:14;:21;27861:5;27846:21;;;;;;;;;;;:29;27868:6;27846:29;;;;;;;;;;;27839:36;;;27892:8;:15;27901:5;27892:15;;;;;;;;;;;:23;27908:6;27892:23;;;;;;;;;;;;27885:30;;;;;;;;:::i;:::-;;;27932:5;27925:12;;;27648:296;;;;:::o;1441:138:6:-;1493:7;1519:47;811:66;1546:19;;1519:26;:47::i;:::-;:53;;;;;;;;;;;;1512:60;;1441:138;:::o;2264:344::-;2355:37;2374:17;2355:18;:37::i;:::-;2425:17;2407:36;;;;;;;;;;;;2472:1;2458:4;:11;:15;2454:148;;;2489:53;2518:17;2537:4;2489:28;:53::i;:::-;;2454:148;;;2573:18;:16;:18::i;:::-;2454:148;2264:344;;:::o;887:96:3:-;940:7;966:10;959:17;;887:96;:::o;1422:633:18:-;1529:7;1548:20;1571:4;1548:27;;1590:9;1585:435;1609:5;:12;1605:1;:16;1585:435;;;1736:15;1754:5;1760:1;1754:8;;;;;;;;:::i;:::-;;;;;;;;1736:26;;1796:1;1791;1780:8;:12;;;;:::i;:::-;:17;1776:207;;1832:41;1851:12;1865:7;1832:18;:41::i;:::-;1817:56;;1776:207;;;1927:41;1946:7;1955:12;1927:18;:41::i;:::-;1912:56;;1776:207;2008:1;1996:13;;;;;:::i;:::-;;;1628:392;1623:3;;;;;;;1585:435;;;;2036:12;2029:19;;;1422:633;;;;;:::o;7084:141:1:-;7151:17;:15;:17::i;:::-;7146:73;;7191:17;;;;;;;;;;;;;;7146:73;7084:141::o;1980:235:0:-;6931:20:1;:18;:20::i;:::-;2100:1:0::1;2076:26;;:12;:26;;::::0;2072:95:::1;;2153:1;2125:31;;;;;;;;;;;:::i;:::-;;;;;;;;2072:95;2176:32;2195:12;2176:18;:32::i;:::-;1980:235:::0;:::o;32605:117:17:-;32668:7;32694:21;32713:1;32705:5;:9;;;;:::i;:::-;32694:10;:21::i;:::-;32687:28;;32605:117;;;:::o;29944:537::-;30031:11;30059:29;30070:10;:17;30081:5;30070:17;;;;;;;;;;;;30059:10;:29::i;:::-;30053:3;:35;;;;:::i;:::-;30031:58;;30099:9;30119:22;30135:5;30119:15;:22::i;:::-;30099:43;;30299:176;30311:3;30306:1;:8;;:37;;;;;30326:10;:17;30337:5;30326:17;;;;;;;;;;;;30318:5;:25;30306:37;30299:176;;;30390:5;30359:13;:20;30373:5;30359:20;;;;;;;;;;;:27;30380:5;30359:27;;;;;;;;;;;;:36;;;;;;;:::i;:::-;;;;;;;;30423:1;30418;:6;;30409:15;;;;;:::i;:::-;;;30442:22;30458:5;30442:15;:22::i;:::-;30438:26;;30299:176;;;30021:460;;29944:537;;;:::o;1899:163:10:-;1960:21;2042:4;2032:14;;1899:163;;;:::o;1671:281:6:-;1781:1;1748:17;:29;;;:34;1744:119;;1834:17;1805:47;;;;;;;;;;;:::i;:::-;;;;;;;;1744:119;1928:17;1872:47;811:66;1899:19;;1872:26;:47::i;:::-;:53;;;:73;;;;;;;;;;;;;;;;;;1671:281;:::o;3900:253:8:-;3983:12;4008;4022:23;4049:6;:19;;4069:4;4049:25;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4007:67;;;;4091:55;4118:6;4126:7;4135:10;4091:26;:55::i;:::-;4084:62;;;;3900:253;;;;:::o;6113:122:6:-;6175:1;6163:9;:13;6159:70;;;6199:19;;;;;;;;;;;;;;6159:70;6113:122::o;11407:121:18:-;11473:7;11499:22;11516:1;11519;11499:16;:22::i;:::-;11492:29;;11407:121;;;;:::o;8487:120:1:-;8537:4;8560:26;:24;:26::i;:::-;:40;;;;;;;;;;;;8553:47;;8487:120;:::o;1606:793:14:-;1653:7;1693:16;1680:1;:30;;1672:77;;;;;;;;;;;;:::i;:::-;;;;;;;;;1759:9;1771:3;1759:15;;1786:8;1805:1;1797:10;;;:::i;:::-;1786:21;;1832:1;1821;:13;1817:17;;1853:1;1848;:6;1844:40;;1870:3;;;;;:::i;:::-;;;;1844:40;1912:1;784:66;1897:1;:11;:16;1893:55;;1934:3;1929:8;;;;;:::i;:::-;;;1893:55;1975:1;881:66;1961:1;:10;:15;1957:53;;1997:2;1992:7;;;;;:::i;:::-;;;1957:53;2037:1;978:66;2023:1;:10;:15;2019:53;;2059:2;2054:7;;;;;:::i;:::-;;;2019:53;2099:1;1075:66;2085:1;:10;:15;2081:53;;2121:2;2116:7;;;;;:::i;:::-;;;2081:53;2160:1;1171:66;2147:1;:9;:14;2143:51;;2182:1;2177:6;;;;;:::i;:::-;;;2143:51;2220:1;1267:66;2207:1;:9;:14;2203:51;;2242:1;2237:6;;;;;:::i;:::-;;;2203:51;2280:1;1363:66;2267:1;:9;:14;2263:51;;2302:1;2297:6;;;;;:::i;:::-;;;2263:51;2340:1;1459:66;2327:1;:9;:14;2323:51;;2362:1;2357:6;;;;;:::i;:::-;;;2323:51;2391:1;2384:8;;;;1606:793;;;:::o;4421:582:8:-;4565:12;4594:7;4589:408;;4617:19;4625:10;4617:7;:19::i;:::-;4589:408;;;4862:1;4841:10;:17;:22;:49;;;;;4889:1;4867:6;:18;;;:23;4841:49;4837:119;;;4934:6;4917:24;;;;;;;;;;;:::i;:::-;;;;;;;;4837:119;4976:10;4969:17;;;;4589:408;4421:582;;;;;;:::o;11645:532:18:-;11715:13;11792:1;11786:4;11779:15;11820:1;11814:4;11807:15;11941:4;11935;11929;11923;11918:3;11911:5;11900:46;11890:102;;11976:1;11973;11966:12;11890:102;12033:4;12027:11;12018:20;;12149:11;12142:5;12138:23;12129:32;;11645:532;;;;:::o;5543:487:8:-;5694:1;5674:10;:17;:21;5670:354;;;5871:10;5865:17;5927:15;5914:10;5910:2;5906:19;5899:44;5670:354;5994:19;;;;;;;;;;;;;;-1:-1:-1;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;:::o;:::-;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;:::o;7:77:20:-;44:7;73:5;62:16;;7:77;;;:::o;90:118::-;177:24;195:5;177:24;:::i;:::-;172:3;165:37;90:118;;:::o;214:222::-;307:4;345:2;334:9;330:18;322:26;;358:71;426:1;415:9;411:17;402:6;358:71;:::i;:::-;214:222;;;;:::o;442:75::-;475:6;508:2;502:9;492:19;;442:75;:::o;523:117::-;632:1;629;622:12;646:117;755:1;752;745:12;769:122;842:24;860:5;842:24;:::i;:::-;835:5;832:35;822:63;;881:1;878;871:12;822:63;769:122;:::o;897:139::-;943:5;981:6;968:20;959:29;;997:33;1024:5;997:33;:::i;:::-;897:139;;;;:::o;1042:117::-;1151:1;1148;1141:12;1165:117;1274:1;1271;1264:12;1288:117;1397:1;1394;1387:12;1428:568;1501:8;1511:6;1561:3;1554:4;1546:6;1542:17;1538:27;1528:122;;1569:79;;:::i;:::-;1528:122;1682:6;1669:20;1659:30;;1712:18;1704:6;1701:30;1698:117;;;1734:79;;:::i;:::-;1698:117;1848:4;1840:6;1836:17;1824:29;;1902:3;1894:4;1886:6;1882:17;1872:8;1868:32;1865:41;1862:128;;;1909:79;;:::i;:::-;1862:128;1428:568;;;;;:::o;2002:704::-;2097:6;2105;2113;2162:2;2150:9;2141:7;2137:23;2133:32;2130:119;;;2168:79;;:::i;:::-;2130:119;2288:1;2313:53;2358:7;2349:6;2338:9;2334:22;2313:53;:::i;:::-;2303:63;;2259:117;2443:2;2432:9;2428:18;2415:32;2474:18;2466:6;2463:30;2460:117;;;2496:79;;:::i;:::-;2460:117;2609:80;2681:7;2672:6;2661:9;2657:22;2609:80;:::i;:::-;2591:98;;;;2386:313;2002:704;;;;;:::o;2712:147::-;2812:6;2846:5;2840:12;2830:22;;2712:147;;;:::o;2865:217::-;2997:11;3031:6;3026:3;3019:19;3071:4;3066:3;3062:14;3047:29;;2865:217;;;;:::o;3088:165::-;3188:4;3211:3;3203:11;;3241:4;3236:3;3232:14;3224:22;;3088:165;;;:::o;3259:108::-;3336:24;3354:5;3336:24;:::i;:::-;3331:3;3324:37;3259:108;;:::o;3453:517::-;3606:4;3601:3;3597:14;3695:4;3688:5;3684:16;3678:23;3714:63;3771:4;3766:3;3762:14;3748:12;3714:63;:::i;:::-;3621:166;3871:4;3864:5;3860:16;3854:23;3890:63;3947:4;3942:3;3938:14;3924:12;3890:63;:::i;:::-;3797:166;3575:395;3453:517;;:::o;3976:311::-;4111:10;4132:112;4240:3;4232:6;4132:112;:::i;:::-;4276:4;4271:3;4267:14;4253:28;;3976:311;;;;:::o;4293:146::-;4396:4;4428;4423:3;4419:14;4411:22;;4293:146;;;:::o;4529:996::-;4714:3;4743:87;4824:5;4743:87;:::i;:::-;4846:119;4958:6;4953:3;4846:119;:::i;:::-;4839:126;;4989:89;5072:5;4989:89;:::i;:::-;5101:7;5132:1;5117:383;5142:6;5139:1;5136:13;5117:383;;;5218:6;5212:13;5245:129;5370:3;5355:13;5245:129;:::i;:::-;5238:136;;5397:93;5483:6;5397:93;:::i;:::-;5387:103;;5177:323;5164:1;5161;5157:9;5152:14;;5117:383;;;5121:14;5516:3;5509:10;;4719:806;;;4529:996;;;;:::o;5531:505::-;5740:4;5778:2;5767:9;5763:18;5755:26;;5827:9;5821:4;5817:20;5813:1;5802:9;5798:17;5791:47;5855:174;6024:4;6015:6;5855:174;:::i;:::-;5847:182;;5531:505;;;;:::o;6042:126::-;6079:7;6119:42;6112:5;6108:54;6097:65;;6042:126;;;:::o;6174:96::-;6211:7;6240:24;6258:5;6240:24;:::i;:::-;6229:35;;6174:96;;;:::o;6276:122::-;6349:24;6367:5;6349:24;:::i;:::-;6342:5;6339:35;6329:63;;6388:1;6385;6378:12;6329:63;6276:122;:::o;6404:139::-;6450:5;6488:6;6475:20;6466:29;;6504:33;6531:5;6504:33;:::i;:::-;6404:139;;;;:::o;6562:552::-;6619:8;6629:6;6679:3;6672:4;6664:6;6660:17;6656:27;6646:122;;6687:79;;:::i;:::-;6646:122;6800:6;6787:20;6777:30;;6830:18;6822:6;6819:30;6816:117;;;6852:79;;:::i;:::-;6816:117;6966:4;6958:6;6954:17;6942:29;;7020:3;7012:4;7004:6;7000:17;6990:8;6986:32;6983:41;6980:128;;;7027:79;;:::i;:::-;6980:128;6562:552;;;;;:::o;7120:672::-;7199:6;7207;7215;7264:2;7252:9;7243:7;7239:23;7235:32;7232:119;;;7270:79;;:::i;:::-;7232:119;7390:1;7415:53;7460:7;7451:6;7440:9;7436:22;7415:53;:::i;:::-;7405:63;;7361:117;7545:2;7534:9;7530:18;7517:32;7576:18;7568:6;7565:30;7562:117;;;7598:79;;:::i;:::-;7562:117;7711:64;7767:7;7758:6;7747:9;7743:22;7711:64;:::i;:::-;7693:82;;;;7488:297;7120:672;;;;;:::o;7798:118::-;7885:24;7903:5;7885:24;:::i;:::-;7880:3;7873:37;7798:118;;:::o;7922:222::-;8015:4;8053:2;8042:9;8038:18;8030:26;;8066:71;8134:1;8123:9;8119:17;8110:6;8066:71;:::i;:::-;7922:222;;;;:::o;8187:596::-;8288:8;8298:6;8348:3;8341:4;8333:6;8329:17;8325:27;8315:122;;8356:79;;:::i;:::-;8315:122;8469:6;8456:20;8446:30;;8499:18;8491:6;8488:30;8485:117;;;8521:79;;:::i;:::-;8485:117;8635:4;8627:6;8623:17;8611:29;;8689:3;8681:4;8673:6;8669:17;8659:8;8655:32;8652:41;8649:128;;;8696:79;;:::i;:::-;8649:128;8187:596;;;;;:::o;8789:1103::-;8932:6;8940;8948;8956;8964;9013:2;9001:9;8992:7;8988:23;8984:32;8981:119;;;9019:79;;:::i;:::-;8981:119;9139:1;9164:53;9209:7;9200:6;9189:9;9185:22;9164:53;:::i;:::-;9154:63;;9110:117;9294:2;9283:9;9279:18;9266:32;9325:18;9317:6;9314:30;9311:117;;;9347:79;;:::i;:::-;9311:117;9460:108;9560:7;9551:6;9540:9;9536:22;9460:108;:::i;:::-;9442:126;;;;9237:341;9645:2;9634:9;9630:18;9617:32;9676:18;9668:6;9665:30;9662:117;;;9698:79;;:::i;:::-;9662:117;9811:64;9867:7;9858:6;9847:9;9843:22;9811:64;:::i;:::-;9793:82;;;;9588:297;8789:1103;;;;;;;;:::o;9898:77::-;9935:7;9964:5;9953:16;;9898:77;;;:::o;9981:118::-;10068:24;10086:5;10068:24;:::i;:::-;10063:3;10056:37;9981:118;;:::o;10105:222::-;10198:4;10236:2;10225:9;10221:18;10213:26;;10249:71;10317:1;10306:9;10302:17;10293:6;10249:71;:::i;:::-;10105:222;;;;:::o;10333:329::-;10392:6;10441:2;10429:9;10420:7;10416:23;10412:32;10409:119;;;10447:79;;:::i;:::-;10409:119;10567:1;10592:53;10637:7;10628:6;10617:9;10613:22;10592:53;:::i;:::-;10582:63;;10538:117;10333:329;;;;:::o;10668:1047::-;10783:6;10791;10799;10807;10815;10864:2;10852:9;10843:7;10839:23;10835:32;10832:119;;;10870:79;;:::i;:::-;10832:119;10990:1;11015:53;11060:7;11051:6;11040:9;11036:22;11015:53;:::i;:::-;11005:63;;10961:117;11145:2;11134:9;11130:18;11117:32;11176:18;11168:6;11165:30;11162:117;;;11198:79;;:::i;:::-;11162:117;11311:80;11383:7;11374:6;11363:9;11359:22;11311:80;:::i;:::-;11293:98;;;;11088:313;11468:2;11457:9;11453:18;11440:32;11499:18;11491:6;11488:30;11485:117;;;11521:79;;:::i;:::-;11485:117;11634:64;11690:7;11681:6;11670:9;11666:22;11634:64;:::i;:::-;11616:82;;;;11411:297;10668:1047;;;;;;;;:::o;11721:474::-;11789:6;11797;11846:2;11834:9;11825:7;11821:23;11817:32;11814:119;;;11852:79;;:::i;:::-;11814:119;11972:1;11997:53;12042:7;12033:6;12022:9;12018:22;11997:53;:::i;:::-;11987:63;;11943:117;12099:2;12125:53;12170:7;12161:6;12150:9;12146:22;12125:53;:::i;:::-;12115:63;;12070:118;11721:474;;;;;:::o;12201:98::-;12252:6;12286:5;12280:12;12270:22;;12201:98;;;:::o;12305:158::-;12378:11;12412:6;12407:3;12400:19;12452:4;12447:3;12443:14;12428:29;;12305:158;;;;:::o;12469:246::-;12550:1;12560:113;12574:6;12571:1;12568:13;12560:113;;;12659:1;12654:3;12650:11;12644:18;12640:1;12635:3;12631:11;12624:39;12596:2;12593:1;12589:10;12584:15;;12560:113;;;12707:1;12698:6;12693:3;12689:16;12682:27;12531:184;12469:246;;;:::o;12721:102::-;12762:6;12813:2;12809:7;12804:2;12797:5;12793:14;12789:28;12779:38;;12721:102;;;:::o;12829:353::-;12905:3;12933:38;12965:5;12933:38;:::i;:::-;12987:60;13040:6;13035:3;12987:60;:::i;:::-;12980:67;;13056:65;13114:6;13109:3;13102:4;13095:5;13091:16;13056:65;:::i;:::-;13146:29;13168:6;13146:29;:::i;:::-;13141:3;13137:39;13130:46;;12909:273;12829:353;;;;:::o;13230:422::-;13341:3;13377:4;13372:3;13368:14;13464:4;13457:5;13453:16;13447:23;13517:3;13511:4;13507:14;13500:4;13495:3;13491:14;13484:38;13543:71;13609:4;13595:12;13543:71;:::i;:::-;13535:79;;13392:233;13642:4;13635:11;;13346:306;13230:422;;;;:::o;13658:357::-;13793:4;13831:2;13820:9;13816:18;13808:26;;13880:9;13874:4;13870:20;13866:1;13855:9;13851:17;13844:47;13908:100;14003:4;13994:6;13908:100;:::i;:::-;13900:108;;13658:357;;;;:::o;14021:817::-;14109:6;14117;14125;14133;14182:2;14170:9;14161:7;14157:23;14153:32;14150:119;;;14188:79;;:::i;:::-;14150:119;14308:1;14333:53;14378:7;14369:6;14358:9;14354:22;14333:53;:::i;:::-;14323:63;;14279:117;14435:2;14461:53;14506:7;14497:6;14486:9;14482:22;14461:53;:::i;:::-;14451:63;;14406:118;14591:2;14580:9;14576:18;14563:32;14622:18;14614:6;14611:30;14608:117;;;14644:79;;:::i;:::-;14608:117;14757:64;14813:7;14804:6;14793:9;14789:22;14757:64;:::i;:::-;14739:82;;;;14534:297;14021:817;;;;;;;:::o;14844:332::-;14965:4;15003:2;14992:9;14988:18;14980:26;;15016:71;15084:1;15073:9;15069:17;15060:6;15016:71;:::i;:::-;15097:72;15165:2;15154:9;15150:18;15141:6;15097:72;:::i;:::-;14844:332;;;;;:::o;15182:90::-;15216:7;15259:5;15252:13;15245:21;15234:32;;15182:90;;;:::o;15278:109::-;15359:21;15374:5;15359:21;:::i;:::-;15354:3;15347:34;15278:109;;:::o;15393:210::-;15480:4;15518:2;15507:9;15503:18;15495:26;;15531:65;15593:1;15582:9;15578:17;15569:6;15531:65;:::i;:::-;15393:210;;;;:::o;15609:117::-;15718:1;15715;15708:12;15732:180;15780:77;15777:1;15770:88;15877:4;15874:1;15867:15;15901:4;15898:1;15891:15;15918:281;16001:27;16023:4;16001:27;:::i;:::-;15993:6;15989:40;16131:6;16119:10;16116:22;16095:18;16083:10;16080:34;16077:62;16074:88;;;16142:18;;:::i;:::-;16074:88;16182:10;16178:2;16171:22;15961:238;15918:281;;:::o;16205:129::-;16239:6;16266:20;;:::i;:::-;16256:30;;16295:33;16323:4;16315:6;16295:33;:::i;:::-;16205:129;;;:::o;16340:307::-;16401:4;16491:18;16483:6;16480:30;16477:56;;;16513:18;;:::i;:::-;16477:56;16551:29;16573:6;16551:29;:::i;:::-;16543:37;;16635:4;16629;16625:15;16617:23;;16340:307;;;:::o;16653:146::-;16750:6;16745:3;16740;16727:30;16791:1;16782:6;16777:3;16773:16;16766:27;16653:146;;;:::o;16805:423::-;16882:5;16907:65;16923:48;16964:6;16923:48;:::i;:::-;16907:65;:::i;:::-;16898:74;;16995:6;16988:5;16981:21;17033:4;17026:5;17022:16;17071:3;17062:6;17057:3;17053:16;17050:25;17047:112;;;17078:79;;:::i;:::-;17047:112;17168:54;17215:6;17210:3;17205;17168:54;:::i;:::-;16888:340;16805:423;;;;;:::o;17247:338::-;17302:5;17351:3;17344:4;17336:6;17332:17;17328:27;17318:122;;17359:79;;:::i;:::-;17318:122;17476:6;17463:20;17501:78;17575:3;17567:6;17560:4;17552:6;17548:17;17501:78;:::i;:::-;17492:87;;17308:277;17247:338;;;;:::o;17591:652::-;17668:6;17676;17725:2;17713:9;17704:7;17700:23;17696:32;17693:119;;;17731:79;;:::i;:::-;17693:119;17851:1;17876:53;17921:7;17912:6;17901:9;17897:22;17876:53;:::i;:::-;17866:63;;17822:117;18006:2;17995:9;17991:18;17978:32;18037:18;18029:6;18026:30;18023:117;;;18059:79;;:::i;:::-;18023:117;18164:62;18218:7;18209:6;18198:9;18194:22;18164:62;:::i;:::-;18154:72;;17949:287;17591:652;;;;;:::o;18249:101::-;18285:7;18325:18;18318:5;18314:30;18303:41;;18249:101;;;:::o;18356:115::-;18441:23;18458:5;18441:23;:::i;:::-;18436:3;18429:36;18356:115;;:::o;18477:90::-;18512:7;18555:5;18552:1;18541:20;18530:31;;18477:90;;;:::o;18573:112::-;18656:22;18672:5;18656:22;:::i;:::-;18651:3;18644:35;18573:112;;:::o;18691:320::-;18806:4;18844:2;18833:9;18829:18;18821:26;;18857:69;18923:1;18912:9;18908:17;18899:6;18857:69;:::i;:::-;18936:68;19000:2;18989:9;18985:18;18976:6;18936:68;:::i;:::-;18691:320;;;;;:::o;19017:60::-;19045:3;19066:5;19059:12;;19017:60;;;:::o;19083:142::-;19133:9;19166:53;19184:34;19193:24;19211:5;19193:24;:::i;:::-;19184:34;:::i;:::-;19166:53;:::i;:::-;19153:66;;19083:142;;;:::o;19231:126::-;19281:9;19314:37;19345:5;19314:37;:::i;:::-;19301:50;;19231:126;;;:::o;19363:140::-;19427:9;19460:37;19491:5;19460:37;:::i;:::-;19447:50;;19363:140;;;:::o;19509:159::-;19610:51;19655:5;19610:51;:::i;:::-;19605:3;19598:64;19509:159;;:::o;19674:250::-;19781:4;19819:2;19808:9;19804:18;19796:26;;19832:85;19914:1;19903:9;19899:17;19890:6;19832:85;:::i;:::-;19674:250;;;;:::o;19930:474::-;19998:6;20006;20055:2;20043:9;20034:7;20030:23;20026:32;20023:119;;;20061:79;;:::i;:::-;20023:119;20181:1;20206:53;20251:7;20242:6;20231:9;20227:22;20206:53;:::i;:::-;20196:63;;20152:117;20308:2;20334:53;20379:7;20370:6;20359:9;20355:22;20334:53;:::i;:::-;20324:63;;20279:118;19930:474;;;;;:::o;20410:114::-;20477:6;20511:5;20505:12;20495:22;;20410:114;;;:::o;20530:184::-;20629:11;20663:6;20658:3;20651:19;20703:4;20698:3;20694:14;20679:29;;20530:184;;;;:::o;20720:132::-;20787:4;20810:3;20802:11;;20840:4;20835:3;20831:14;20823:22;;20720:132;;;:::o;20858:179::-;20927:10;20948:46;20990:3;20982:6;20948:46;:::i;:::-;21026:4;21021:3;21017:14;21003:28;;20858:179;;;;:::o;21043:113::-;21113:4;21145;21140:3;21136:14;21128:22;;21043:113;;;:::o;21192:732::-;21311:3;21340:54;21388:5;21340:54;:::i;:::-;21410:86;21489:6;21484:3;21410:86;:::i;:::-;21403:93;;21520:56;21570:5;21520:56;:::i;:::-;21599:7;21630:1;21615:284;21640:6;21637:1;21634:13;21615:284;;;21716:6;21710:13;21743:63;21802:3;21787:13;21743:63;:::i;:::-;21736:70;;21829:60;21882:6;21829:60;:::i;:::-;21819:70;;21675:224;21662:1;21659;21655:9;21650:14;;21615:284;;;21619:14;21915:3;21908:10;;21316:608;;;21192:732;;;;:::o;21930:373::-;22073:4;22111:2;22100:9;22096:18;22088:26;;22160:9;22154:4;22150:20;22146:1;22135:9;22131:17;22124:47;22188:108;22291:4;22282:6;22188:108;:::i;:::-;22180:116;;21930:373;;;;:::o;22309:672::-;22388:6;22396;22404;22453:2;22441:9;22432:7;22428:23;22424:32;22421:119;;;22459:79;;:::i;:::-;22421:119;22579:1;22604:53;22649:7;22640:6;22629:9;22625:22;22604:53;:::i;:::-;22594:63;;22550:117;22734:2;22723:9;22719:18;22706:32;22765:18;22757:6;22754:30;22751:117;;;22787:79;;:::i;:::-;22751:117;22900:64;22956:7;22947:6;22936:9;22932:22;22900:64;:::i;:::-;22882:82;;;;22677:297;22309:672;;;;;:::o;22987:218::-;23078:4;23116:2;23105:9;23101:18;23093:26;;23129:69;23195:1;23184:9;23180:17;23171:6;23129:69;:::i;:::-;22987:218;;;;:::o;23211:99::-;23263:6;23297:5;23291:12;23281:22;;23211:99;;;:::o;23316:169::-;23400:11;23434:6;23429:3;23422:19;23474:4;23469:3;23465:14;23450:29;;23316:169;;;;:::o;23491:377::-;23579:3;23607:39;23640:5;23607:39;:::i;:::-;23662:71;23726:6;23721:3;23662:71;:::i;:::-;23655:78;;23742:65;23800:6;23795:3;23788:4;23781:5;23777:16;23742:65;:::i;:::-;23832:29;23854:6;23832:29;:::i;:::-;23827:3;23823:39;23816:46;;23583:285;23491:377;;;;:::o;23874:313::-;23987:4;24025:2;24014:9;24010:18;24002:26;;24074:9;24068:4;24064:20;24060:1;24049:9;24045:17;24038:47;24102:78;24175:4;24166:6;24102:78;:::i;:::-;24094:86;;23874:313;;;;:::o;24193:329::-;24252:6;24301:2;24289:9;24280:7;24276:23;24272:32;24269:119;;;24307:79;;:::i;:::-;24269:119;24427:1;24452:53;24497:7;24488:6;24477:9;24473:22;24452:53;:::i;:::-;24442:63;;24398:117;24193:329;;;;:::o;24562:593::-;24660:8;24670:6;24720:3;24713:4;24705:6;24701:17;24697:27;24687:122;;24728:79;;:::i;:::-;24687:122;24841:6;24828:20;24818:30;;24871:18;24863:6;24860:30;24857:117;;;24893:79;;:::i;:::-;24857:117;25007:4;24999:6;24995:17;24983:29;;25061:3;25053:4;25045:6;25041:17;25031:8;25027:32;25024:41;25021:128;;;25068:79;;:::i;:::-;25021:128;24562:593;;;;;:::o;25161:754::-;25281:6;25289;25297;25346:2;25334:9;25325:7;25321:23;25317:32;25314:119;;;25352:79;;:::i;:::-;25314:119;25472:1;25497:53;25542:7;25533:6;25522:9;25518:22;25497:53;:::i;:::-;25487:63;;25443:117;25627:2;25616:9;25612:18;25599:32;25658:18;25650:6;25647:30;25644:117;;;25680:79;;:::i;:::-;25644:117;25793:105;25890:7;25881:6;25870:9;25866:22;25793:105;:::i;:::-;25775:123;;;;25570:338;25161:754;;;;;:::o;25921:180::-;25969:77;25966:1;25959:88;26066:4;26063:1;26056:15;26090:4;26087:1;26080:15;26107:194;26147:4;26167:20;26185:1;26167:20;:::i;:::-;26162:25;;26201:20;26219:1;26201:20;:::i;:::-;26196:25;;26245:1;26242;26238:9;26230:17;;26269:1;26263:4;26260:11;26257:37;;;26274:18;;:::i;:::-;26257:37;26107:194;;;;:::o;26307:180::-;26355:77;26352:1;26345:88;26452:4;26449:1;26442:15;26476:4;26473:1;26466:15;26493:170;26633:22;26629:1;26621:6;26617:14;26610:46;26493:170;:::o;26669:366::-;26811:3;26832:67;26896:2;26891:3;26832:67;:::i;:::-;26825:74;;26908:93;26997:3;26908:93;:::i;:::-;27026:2;27021:3;27017:12;27010:19;;26669:366;;;:::o;27041:419::-;27207:4;27245:2;27234:9;27230:18;27222:26;;27294:9;27288:4;27284:20;27280:1;27269:9;27265:17;27258:47;27322:131;27448:4;27322:131;:::i;:::-;27314:139;;27041:419;;;:::o;27466:167::-;27606:19;27602:1;27594:6;27590:14;27583:43;27466:167;:::o;27639:366::-;27781:3;27802:67;27866:2;27861:3;27802:67;:::i;:::-;27795:74;;27878:93;27967:3;27878:93;:::i;:::-;27996:2;27991:3;27987:12;27980:19;;27639:366;;;:::o;28011:419::-;28177:4;28215:2;28204:9;28200:18;28192:26;;28264:9;28258:4;28254:20;28250:1;28239:9;28235:17;28228:47;28292:131;28418:4;28292:131;:::i;:::-;28284:139;;28011:419;;;:::o;28436:183::-;28474:3;28497:23;28514:5;28497:23;:::i;:::-;28488:32;;28542:18;28535:5;28532:29;28529:55;;28564:18;;:::i;:::-;28529:55;28611:1;28604:5;28600:13;28593:20;;28436:183;;;:::o;28625:168::-;28708:11;28742:6;28737:3;28730:19;28782:4;28777:3;28773:14;28758:29;;28625:168;;;;:::o;28821:314::-;28917:3;28938:70;29001:6;28996:3;28938:70;:::i;:::-;28931:77;;29018:56;29067:6;29062:3;29055:5;29018:56;:::i;:::-;29099:29;29121:6;29099:29;:::i;:::-;29094:3;29090:39;29083:46;;28821:314;;;;;:::o;29141:549::-;29318:4;29356:2;29345:9;29341:18;29333:26;;29369:71;29437:1;29426:9;29422:17;29413:6;29369:71;:::i;:::-;29450:72;29518:2;29507:9;29503:18;29494:6;29450:72;:::i;:::-;29569:9;29563:4;29559:20;29554:2;29543:9;29539:18;29532:48;29597:86;29678:4;29669:6;29661;29597:86;:::i;:::-;29589:94;;29141:549;;;;;;;:::o;29696:168::-;29836:20;29832:1;29824:6;29820:14;29813:44;29696:168;:::o;29870:366::-;30012:3;30033:67;30097:2;30092:3;30033:67;:::i;:::-;30026:74;;30109:93;30198:3;30109:93;:::i;:::-;30227:2;30222:3;30218:12;30211:19;;29870:366;;;:::o;30242:419::-;30408:4;30446:2;30435:9;30431:18;30423:26;;30495:9;30489:4;30485:20;30481:1;30470:9;30466:17;30459:47;30523:131;30649:4;30523:131;:::i;:::-;30515:139;;30242:419;;;:::o;30667:176::-;30807:28;30803:1;30795:6;30791:14;30784:52;30667:176;:::o;30849:366::-;30991:3;31012:67;31076:2;31071:3;31012:67;:::i;:::-;31005:74;;31088:93;31177:3;31088:93;:::i;:::-;31206:2;31201:3;31197:12;31190:19;;30849:366;;;:::o;31221:419::-;31387:4;31425:2;31414:9;31410:18;31402:26;;31474:9;31468:4;31464:20;31460:1;31449:9;31445:17;31438:47;31502:131;31628:4;31502:131;:::i;:::-;31494:139;;31221:419;;;:::o;31646:178::-;31786:30;31782:1;31774:6;31770:14;31763:54;31646:178;:::o;31830:366::-;31972:3;31993:67;32057:2;32052:3;31993:67;:::i;:::-;31986:74;;32069:93;32158:3;32069:93;:::i;:::-;32187:2;32182:3;32178:12;32171:19;;31830:366;;;:::o;32202:419::-;32368:4;32406:2;32395:9;32391:18;32383:26;;32455:9;32449:4;32445:20;32441:1;32430:9;32426:17;32419:47;32483:131;32609:4;32483:131;:::i;:::-;32475:139;;32202:419;;;:::o;32627:117::-;32736:1;32733;32726:12;32750:117;32859:1;32856;32849:12;32873:117;32982:1;32979;32972:12;32996:394;33090:4;33144:11;33131:25;33244:1;33238:4;33234:12;33223:8;33207:14;33203:29;33199:48;33179:18;33175:73;33165:168;;33252:79;;:::i;:::-;33165:168;33364:18;33354:8;33350:33;33342:41;;33095:295;32996:394;;;;:::o;33396:389::-;33485:4;33539:11;33526:25;33639:1;33633:4;33629:12;33618:8;33602:14;33598:29;33594:48;33574:18;33570:73;33560:168;;33647:79;;:::i;:::-;33560:168;33759:18;33749:8;33745:33;33737:41;;33490:295;33396:389;;;;:::o;33791:191::-;33831:3;33850:20;33868:1;33850:20;:::i;:::-;33845:25;;33884:20;33902:1;33884:20;:::i;:::-;33879:25;;33927:1;33924;33920:9;33913:16;;33948:3;33945:1;33942:10;33939:36;;;33955:18;;:::i;:::-;33939:36;33791:191;;;;:::o;33988:210::-;34113:11;34147:6;34142:3;34135:19;34187:4;34182:3;34178:14;34163:29;;33988:210;;;;:::o;34204:130::-;34301:4;34324:3;34316:11;;34204:130;;;:::o;34340:117::-;34449:1;34446;34439:12;34463:370;34539:5;34594:3;34581:17;34686:1;34680:4;34676:12;34665:8;34649:14;34645:29;34641:48;34621:18;34617:73;34607:168;;34694:79;;:::i;:::-;34607:168;34817:8;34797:18;34793:33;34784:42;;34545:288;34463:370;;;;:::o;34839:117::-;34948:1;34945;34938:12;34962:117;35071:1;35068;35061:12;35085:711;35149:5;35156:6;35212:3;35199:17;35304:1;35298:4;35294:12;35283:8;35267:14;35263:29;35259:48;35239:18;35235:73;35225:168;;35312:79;;:::i;:::-;35225:168;35435:8;35415:18;35411:33;35402:42;;35477:5;35464:19;35454:29;;35512:4;35505:5;35501:16;35492:25;;35540:18;35532:6;35529:30;35526:117;;;35562:79;;:::i;:::-;35526:117;35698:4;35690:6;35686:17;35670:14;35666:38;35659:5;35655:50;35652:137;;;35708:79;;:::i;:::-;35652:137;35163:633;35085:711;;;;;:::o;35824:294::-;35910:3;35931:60;35984:6;35979:3;35931:60;:::i;:::-;35924:67;;36001:56;36050:6;36045:3;36038:5;36001:56;:::i;:::-;36082:29;36104:6;36082:29;:::i;:::-;36077:3;36073:39;36066:46;;35824:294;;;;;:::o;36166:482::-;36269:3;36305:4;36300:3;36296:14;36389:61;36444:4;36437:5;36433:16;36426:5;36389:61;:::i;:::-;36497:3;36491:4;36487:14;36480:4;36475:3;36471:14;36464:38;36523:87;36605:4;36591:12;36577;36523:87;:::i;:::-;36515:95;;36320:301;;36638:4;36631:11;;36274:374;36166:482;;;;:::o;36654:122::-;36706:5;36731:39;36766:2;36761:3;36757:12;36752:3;36731:39;:::i;:::-;36722:48;;36654:122;;;;:::o;36848:704::-;36961:3;36997:4;36992:3;36988:14;37067:73;37134:4;37127:5;37123:16;37116:5;37067:73;:::i;:::-;37187:3;37181:4;37177:14;37170:4;37165:3;37161:14;37154:38;37213:97;37305:4;37291:12;37213:97;:::i;:::-;37205:105;;37012:309;37389:50;37433:4;37426:5;37422:16;37415:5;37389:50;:::i;:::-;37452:63;37509:4;37504:3;37500:14;37486:12;37452:63;:::i;:::-;37331:194;37542:4;37535:11;;36966:586;36848:704;;;;:::o;37558:264::-;37681:10;37716:100;37812:3;37804:6;37716:100;:::i;:::-;37702:114;;37558:264;;;;:::o;37828:375::-;37909:5;37964:3;37951:17;38056:1;38050:4;38046:12;38035:8;38019:14;38015:29;38011:48;37991:18;37987:73;37977:168;;38064:79;;:::i;:::-;37977:168;38187:8;38167:18;38163:33;38154:42;;37915:288;37828:375;;;;:::o;38209:143::-;38309:4;38341;38336:3;38332:14;38324:22;;38209:143;;;:::o;38428:1096::-;38611:3;38634:112;38739:6;38734:3;38634:112;:::i;:::-;38627:119;;38772:3;38817:4;38809:6;38805:17;38800:3;38796:27;38847:86;38927:5;38847:86;:::i;:::-;38956:7;38987:1;38972:507;38997:6;38994:1;38991:13;38972:507;;;39068:9;39062:4;39058:20;39053:3;39046:33;39113:70;39176:6;39167:7;39113:70;:::i;:::-;39204:118;39317:4;39302:13;39204:118;:::i;:::-;39196:126;;39345:90;39428:6;39345:90;:::i;:::-;39335:100;;39464:4;39459:3;39455:14;39448:21;;39032:447;39019:1;39016;39012:9;39007:14;;38972:507;;;38976:14;39495:4;39488:11;;39515:3;39508:10;;38616:908;;;;38428:1096;;;;;:::o;39530:939::-;39849:4;39887:3;39876:9;39872:19;39864:27;;39901:71;39969:1;39958:9;39954:17;39945:6;39901:71;:::i;:::-;39982:72;40050:2;40039:9;40035:18;40026:6;39982:72;:::i;:::-;40101:9;40095:4;40091:20;40086:2;40075:9;40071:18;40064:48;40129:172;40296:4;40287:6;40279;40129:172;:::i;:::-;40121:180;;40348:9;40342:4;40338:20;40333:2;40322:9;40318:18;40311:48;40376:86;40457:4;40448:6;40440;40376:86;:::i;:::-;40368:94;;39530:939;;;;;;;;;:::o;40475:231::-;40615:34;40611:1;40603:6;40599:14;40592:58;40684:14;40679:2;40671:6;40667:15;40660:39;40475:231;:::o;40712:366::-;40854:3;40875:67;40939:2;40934:3;40875:67;:::i;:::-;40868:74;;40951:93;41040:3;40951:93;:::i;:::-;41069:2;41064:3;41060:12;41053:19;;40712:366;;;:::o;41084:419::-;41250:4;41288:2;41277:9;41273:18;41265:26;;41337:9;41331:4;41327:20;41323:1;41312:9;41308:17;41301:47;41365:131;41491:4;41365:131;:::i;:::-;41357:139;;41084:419;;;:::o;41509:245::-;41649:34;41645:1;41637:6;41633:14;41626:58;41718:28;41713:2;41705:6;41701:15;41694:53;41509:245;:::o;41760:366::-;41902:3;41923:67;41987:2;41982:3;41923:67;:::i;:::-;41916:74;;41999:93;42088:3;41999:93;:::i;:::-;42117:2;42112:3;42108:12;42101:19;;41760:366;;;:::o;42132:419::-;42298:4;42336:2;42325:9;42321:18;42313:26;;42385:9;42379:4;42375:20;42371:1;42360:9;42356:17;42349:47;42413:131;42539:4;42413:131;:::i;:::-;42405:139;;42132:419;;;:::o;42557:230::-;42697:34;42693:1;42685:6;42681:14;42674:58;42766:13;42761:2;42753:6;42749:15;42742:38;42557:230;:::o;42793:366::-;42935:3;42956:67;43020:2;43015:3;42956:67;:::i;:::-;42949:74;;43032:93;43121:3;43032:93;:::i;:::-;43150:2;43145:3;43141:12;43134:19;;42793:366;;;:::o;43165:419::-;43331:4;43369:2;43358:9;43354:18;43346:26;;43418:9;43412:4;43408:20;43404:1;43393:9;43389:17;43382:47;43446:131;43572:4;43446:131;:::i;:::-;43438:139;;43165:419;;;:::o;43590:117::-;43699:1;43696;43689:12;43713:98;43797:6;43792:3;43787;43774:30;43713:98;;;:::o;43847:537::-;43975:3;43996:86;44075:6;44070:3;43996:86;:::i;:::-;43989:93;;44106:66;44098:6;44095:78;44092:165;;;44176:79;;:::i;:::-;44092:165;44288:4;44280:6;44276:17;44266:27;;44303:43;44339:6;44334:3;44327:5;44303:43;:::i;:::-;44371:6;44366:3;44362:16;44355:23;;43847:537;;;;;:::o;44390:720::-;44627:4;44665:2;44654:9;44650:18;44642:26;;44678:71;44746:1;44735:9;44731:17;44722:6;44678:71;:::i;:::-;44796:9;44790:4;44786:20;44781:2;44770:9;44766:18;44759:48;44824:118;44937:4;44928:6;44920;44824:118;:::i;:::-;44816:126;;44989:9;44983:4;44979:20;44974:2;44963:9;44959:18;44952:48;45017:86;45098:4;45089:6;45081;45017:86;:::i;:::-;45009:94;;44390:720;;;;;;;;:::o;45116:180::-;45164:77;45161:1;45154:88;45261:4;45258:1;45251:15;45285:4;45282:1;45275:15;45302:320;45346:6;45383:1;45377:4;45373:12;45363:22;;45430:1;45424:4;45420:12;45451:18;45441:81;;45507:4;45499:6;45495:17;45485:27;;45441:81;45569:2;45561:6;45558:14;45538:18;45535:38;45532:84;;45588:18;;:::i;:::-;45532:84;45353:269;45302:320;;;:::o;45628:79::-;45667:7;45696:5;45685:16;;45628:79;;;:::o;45713:157::-;45818:45;45838:24;45856:5;45838:24;:::i;:::-;45818:45;:::i;:::-;45813:3;45806:58;45713:157;;:::o;45876:256::-;45988:3;46003:75;46074:3;46065:6;46003:75;:::i;:::-;46103:2;46098:3;46094:12;46087:19;;46123:3;46116:10;;45876:256;;;;:::o;46138:147::-;46239:11;46276:3;46261:18;;46138:147;;;;:::o;46291:386::-;46395:3;46423:38;46455:5;46423:38;:::i;:::-;46477:88;46558:6;46553:3;46477:88;:::i;:::-;46470:95;;46574:65;46632:6;46627:3;46620:4;46613:5;46609:16;46574:65;:::i;:::-;46664:6;46659:3;46655:16;46648:23;;46399:278;46291:386;;;;:::o;46683:271::-;46813:3;46835:93;46924:3;46915:6;46835:93;:::i;:::-;46828:100;;46945:3;46938:10;;46683:271;;;;:::o;46960:220::-;47100:34;47096:1;47088:6;47084:14;47077:58;47169:3;47164:2;47156:6;47152:15;47145:28;46960:220;:::o;47186:366::-;47328:3;47349:67;47413:2;47408:3;47349:67;:::i;:::-;47342:74;;47425:93;47514:3;47425:93;:::i;:::-;47543:2;47538:3;47534:12;47527:19;;47186:366;;;:::o;47558:419::-;47724:4;47762:2;47751:9;47747:18;47739:26;;47811:9;47805:4;47801:20;47797:1;47786:9;47782:17;47775:47;47839:131;47965:4;47839:131;:::i;:::-;47831:139;;47558:419;;;:::o;47983:143::-;48040:5;48071:6;48065:13;48056:22;;48087:33;48114:5;48087:33;:::i;:::-;47983:143;;;;:::o;48132:351::-;48202:6;48251:2;48239:9;48230:7;48226:23;48222:32;48219:119;;;48257:79;;:::i;:::-;48219:119;48377:1;48402:64;48458:7;48449:6;48438:9;48434:22;48402:64;:::i;:::-;48392:74;;48348:128;48132:351;;;;:::o;48489:233::-;48629:34;48625:1;48617:6;48613:14;48606:58;48698:16;48693:2;48685:6;48681:15;48674:41;48489:233;:::o;48728:366::-;48870:3;48891:67;48955:2;48950:3;48891:67;:::i;:::-;48884:74;;48967:93;49056:3;48967:93;:::i;:::-;49085:2;49080:3;49076:12;49069:19;;48728:366;;;:::o;49100:419::-;49266:4;49304:2;49293:9;49289:18;49281:26;;49353:9;49347:4;49343:20;49339:1;49328:9;49324:17;49317:47;49381:131;49507:4;49381:131;:::i;:::-;49373:139;;49100:419;;;:::o;49525:231::-;49665:34;49661:1;49653:6;49649:14;49642:58;49734:14;49729:2;49721:6;49717:15;49710:39;49525:231;:::o;49762:366::-;49904:3;49925:67;49989:2;49984:3;49925:67;:::i;:::-;49918:74;;50001:93;50090:3;50001:93;:::i;:::-;50119:2;50114:3;50110:12;50103:19;;49762:366;;;:::o;50134:419::-;50300:4;50338:2;50327:9;50323:18;50315:26;;50387:9;50381:4;50377:20;50373:1;50362:9;50358:17;50351:47;50415:131;50541:4;50415:131;:::i;:::-;50407:139;;50134:419;;;:::o;50559:180::-;50607:77;50604:1;50597:88;50704:4;50701:1;50694:15;50728:4;50725:1;50718:15;50745:295;50885:34;50881:1;50873:6;50869:14;50862:58;50954:34;50949:2;50941:6;50937:15;50930:59;51023:9;51018:2;51010:6;51006:15;50999:34;50745:295;:::o;51046:366::-;51188:3;51209:67;51273:2;51268:3;51209:67;:::i;:::-;51202:74;;51285:93;51374:3;51285:93;:::i;:::-;51403:2;51398:3;51394:12;51387:19;;51046:366;;;:::o;51418:419::-;51584:4;51622:2;51611:9;51607:18;51599:26;;51671:9;51665:4;51661:20;51657:1;51646:9;51642:17;51635:47;51699:131;51825:4;51699:131;:::i;:::-;51691:139;;51418:419;;;:::o;51843:660::-;52048:4;52086:3;52075:9;52071:19;52063:27;;52100:71;52168:1;52157:9;52153:17;52144:6;52100:71;:::i;:::-;52181:72;52249:2;52238:9;52234:18;52225:6;52181:72;:::i;:::-;52263;52331:2;52320:9;52316:18;52307:6;52263:72;:::i;:::-;52382:9;52376:4;52372:20;52367:2;52356:9;52352:18;52345:48;52410:86;52491:4;52482:6;52474;52410:86;:::i;:::-;52402:94;;51843:660;;;;;;;;:::o;52509:332::-;52630:4;52668:2;52657:9;52653:18;52645:26;;52681:71;52749:1;52738:9;52734:17;52725:6;52681:71;:::i;:::-;52762:72;52830:2;52819:9;52815:18;52806:6;52762:72;:::i;:::-;52509:332;;;;;:::o;52847:410::-;52887:7;52910:20;52928:1;52910:20;:::i;:::-;52905:25;;52944:20;52962:1;52944:20;:::i;:::-;52939:25;;52999:1;52996;52992:9;53021:30;53039:11;53021:30;:::i;:::-;53010:41;;53200:1;53191:7;53187:15;53184:1;53181:22;53161:1;53154:9;53134:83;53111:139;;53230:18;;:::i;:::-;53111:139;52895:362;52847:410;;;;:::o;53263:332::-;53384:4;53422:2;53411:9;53407:18;53399:26;;53435:71;53503:1;53492:9;53488:17;53479:6;53435:71;:::i;:::-;53516:72;53584:2;53573:9;53569:18;53560:6;53516:72;:::i;:::-;53263:332;;;;;:::o;53601:117::-;53710:1;53707;53700:12;53847:90;53882:7;53925:5;53922:1;53911:20;53900:31;;53847:90;;;:::o;53943:118::-;54014:22;54030:5;54014:22;:::i;:::-;54007:5;54004:33;53994:61;;54051:1;54048;54041:12;53994:61;53943:118;:::o;54067:139::-;54122:5;54153:6;54147:13;54138:22;;54169:31;54194:5;54169:31;:::i;:::-;54067:139;;;;:::o;54212:120::-;54284:23;54301:5;54284:23;:::i;:::-;54277:5;54274:34;54264:62;;54322:1;54319;54312:12;54264:62;54212:120;:::o;54338:141::-;54394:5;54425:6;54419:13;54410:22;;54441:32;54467:5;54441:32;:::i;:::-;54338:141;;;;:::o;54485:118::-;54556:22;54572:5;54556:22;:::i;:::-;54549:5;54546:33;54536:61;;54593:1;54590;54583:12;54536:61;54485:118;:::o;54609:139::-;54664:5;54695:6;54689:13;54680:22;;54711:31;54736:5;54711:31;:::i;:::-;54609:139;;;;:::o;54786:952::-;54869:5;54913:4;54901:9;54896:3;54892:19;54888:30;54885:117;;;54921:79;;:::i;:::-;54885:117;55020:21;55036:4;55020:21;:::i;:::-;55011:30;;55101:1;55141:58;55195:3;55186:6;55175:9;55171:22;55141:58;:::i;:::-;55134:4;55127:5;55123:16;55116:84;55051:160;55270:2;55311:59;55366:3;55357:6;55346:9;55342:22;55311:59;:::i;:::-;55304:4;55297:5;55293:16;55286:85;55221:161;55441:2;55482:58;55536:3;55527:6;55516:9;55512:22;55482:58;:::i;:::-;55475:4;55468:5;55464:16;55457:84;55392:160;55618:2;55659:60;55715:3;55706:6;55695:9;55691:22;55659:60;:::i;:::-;55652:4;55645:5;55641:16;55634:86;55562:169;54786:952;;;;:::o;55744:398::-;55837:6;55886:3;55874:9;55865:7;55861:23;55857:33;55854:120;;;55893:79;;:::i;:::-;55854:120;56013:1;56038:87;56117:7;56108:6;56097:9;56093:22;56038:87;:::i;:::-;56028:97;;55984:151;55744:398;;;;:::o;56148:235::-;56288:34;56284:1;56276:6;56272:14;56265:58;56357:18;56352:2;56344:6;56340:15;56333:43;56148:235;:::o;56389:366::-;56531:3;56552:67;56616:2;56611:3;56552:67;:::i;:::-;56545:74;;56628:93;56717:3;56628:93;:::i;:::-;56746:2;56741:3;56737:12;56730:19;;56389:366;;;:::o;56761:419::-;56927:4;56965:2;56954:9;56950:18;56942:26;;57014:9;57008:4;57004:20;57000:1;56989:9;56985:17;56978:47;57042:131;57168:4;57042:131;:::i;:::-;57034:139;;56761:419;;;:::o;57186:233::-;57326:34;57322:1;57314:6;57310:14;57303:58;57395:16;57390:2;57382:6;57378:15;57371:41;57186:233;:::o;57425:366::-;57567:3;57588:67;57652:2;57647:3;57588:67;:::i;:::-;57581:74;;57664:93;57753:3;57664:93;:::i;:::-;57782:2;57777:3;57773:12;57766:19;;57425:366;;;:::o;57797:419::-;57963:4;58001:2;57990:9;57986:18;57978:26;;58050:9;58044:4;58040:20;58036:1;58025:9;58021:17;58014:47;58078:131;58204:4;58078:131;:::i;:::-;58070:139;;57797:419;;;:::o;58222:251::-;58362:34;58358:1;58350:6;58346:14;58339:58;58431:34;58426:2;58418:6;58414:15;58407:59;58222:251;:::o;58479:366::-;58621:3;58642:67;58706:2;58701:3;58642:67;:::i;:::-;58635:74;;58718:93;58807:3;58718:93;:::i;:::-;58836:2;58831:3;58827:12;58820:19;;58479:366;;;:::o;58851:419::-;59017:4;59055:2;59044:9;59040:18;59032:26;;59104:9;59098:4;59094:20;59090:1;59079:9;59075:17;59068:47;59132:131;59258:4;59132:131;:::i;:::-;59124:139;;58851:419;;;:::o;59276:176::-;59416:28;59412:1;59404:6;59400:14;59393:52;59276:176;:::o;59458:366::-;59600:3;59621:67;59685:2;59680:3;59621:67;:::i;:::-;59614:74;;59697:93;59786:3;59697:93;:::i;:::-;59815:2;59810:3;59806:12;59799:19;;59458:366;;;:::o;59830:419::-;59996:4;60034:2;60023:9;60019:18;60011:26;;60083:9;60077:4;60073:20;60069:1;60058:9;60054:17;60047:47;60111:131;60237:4;60111:131;:::i;:::-;60103:139;;59830:419;;;:::o;60255:223::-;60395:34;60391:1;60383:6;60379:14;60372:58;60464:6;60459:2;60451:6;60447:15;60440:31;60255:223;:::o;60484:366::-;60626:3;60647:67;60711:2;60706:3;60647:67;:::i;:::-;60640:74;;60723:93;60812:3;60723:93;:::i;:::-;60841:2;60836:3;60832:12;60825:19;;60484:366;;;:::o;60856:419::-;61022:4;61060:2;61049:9;61045:18;61037:26;;61109:9;61103:4;61099:20;61095:1;61084:9;61080:17;61073:47;61137:131;61263:4;61137:131;:::i;:::-;61129:139;;60856:419;;;:::o;61281:549::-;61458:4;61496:2;61485:9;61481:18;61473:26;;61509:71;61577:1;61566:9;61562:17;61553:6;61509:71;:::i;:::-;61590:72;61658:2;61647:9;61643:18;61634:6;61590:72;:::i;:::-;61709:9;61703:4;61699:20;61694:2;61683:9;61679:18;61672:48;61737:86;61818:4;61809:6;61801;61737:86;:::i;:::-;61729:94;;61281:549;;;;;;;:::o;61836:230::-;61976:34;61972:1;61964:6;61960:14;61953:58;62045:13;62040:2;62032:6;62028:15;62021:38;61836:230;:::o;62072:366::-;62214:3;62235:67;62299:2;62294:3;62235:67;:::i;:::-;62228:74;;62311:93;62400:3;62311:93;:::i;:::-;62429:2;62424:3;62420:12;62413:19;;62072:366;;;:::o;62444:419::-;62610:4;62648:2;62637:9;62633:18;62625:26;;62697:9;62691:4;62687:20;62683:1;62672:9;62668:17;62661:47;62725:131;62851:4;62725:131;:::i;:::-;62717:139;;62444:419;;;:::o;62869:165::-;63009:17;63005:1;62997:6;62993:14;62986:41;62869:165;:::o;63040:366::-;63182:3;63203:67;63267:2;63262:3;63203:67;:::i;:::-;63196:74;;63279:93;63368:3;63279:93;:::i;:::-;63397:2;63392:3;63388:12;63381:19;;63040:366;;;:::o;63412:419::-;63578:4;63616:2;63605:9;63601:18;63593:26;;63665:9;63659:4;63655:20;63651:1;63640:9;63636:17;63629:47;63693:131;63819:4;63693:131;:::i;:::-;63685:139;;63412:419;;;:::o;63837:161::-;63977:13;63973:1;63965:6;63961:14;63954:37;63837:161;:::o;64004:366::-;64146:3;64167:67;64231:2;64226:3;64167:67;:::i;:::-;64160:74;;64243:93;64332:3;64243:93;:::i;:::-;64361:2;64356:3;64352:12;64345:19;;64004:366;;;:::o;64376:419::-;64542:4;64580:2;64569:9;64565:18;64557:26;;64629:9;64623:4;64619:20;64615:1;64604:9;64600:17;64593:47;64657:131;64783:4;64657:131;:::i;:::-;64649:139;;64376:419;;;:::o;64801:172::-;64941:24;64937:1;64929:6;64925:14;64918:48;64801:172;:::o;64979:366::-;65121:3;65142:67;65206:2;65201:3;65142:67;:::i;:::-;65135:74;;65218:93;65307:3;65218:93;:::i;:::-;65336:2;65331:3;65327:12;65320:19;;64979:366;;;:::o;65351:419::-;65517:4;65555:2;65544:9;65540:18;65532:26;;65604:9;65598:4;65594:20;65590:1;65579:9;65575:17;65568:47;65632:131;65758:4;65632:131;:::i;:::-;65624:139;;65351:419;;;:::o;65776:96::-;65810:8;65859:5;65854:3;65850:15;65829:36;;65776:96;;;:::o;65878:94::-;65916:7;65945:21;65960:5;65945:21;:::i;:::-;65934:32;;65878:94;;;:::o;65978:153::-;66081:43;66100:23;66117:5;66100:23;:::i;:::-;66081:43;:::i;:::-;66076:3;66069:56;65978:153;;:::o;66137:533::-;66303:3;66318:75;66389:3;66380:6;66318:75;:::i;:::-;66418:2;66413:3;66409:12;66402:19;;66431:75;66502:3;66493:6;66431:75;:::i;:::-;66531:2;66526:3;66522:12;66515:19;;66544:73;66613:3;66604:6;66544:73;:::i;:::-;66642:1;66637:3;66633:11;66626:18;;66661:3;66654:10;;66137:533;;;;;;:::o;66676:180::-;66724:77;66721:1;66714:88;66821:4;66818:1;66811:15;66845:4;66842:1;66835:15;66862:176;66894:1;66911:20;66929:1;66911:20;:::i;:::-;66906:25;;66945:20;66963:1;66945:20;:::i;:::-;66940:25;;66984:1;66974:35;;66989:18;;:::i;:::-;66974:35;67030:1;67027;67023:9;67018:14;;66862:176;;;;:::o;67044:391::-;67135:4;67189:11;67176:25;67289:1;67283:4;67279:12;67268:8;67252:14;67248:29;67244:48;67224:18;67220:73;67210:168;;67297:79;;:::i;:::-;67210:168;67409:18;67399:8;67395:33;67387:41;;67140:295;67044:391;;;;:::o;67441:740::-;67534:4;67540:6;67596:11;67583:25;67696:1;67690:4;67686:12;67675:8;67659:14;67655:29;67651:48;67631:18;67627:73;67617:168;;67704:79;;:::i;:::-;67617:168;67816:18;67806:8;67802:33;67794:41;;67868:4;67855:18;67845:28;;67896:18;67888:6;67885:30;67882:117;;;67918:79;;:::i;:::-;67882:117;68026:2;68020:4;68016:13;68008:21;;68083:4;68075:6;68071:17;68055:14;68051:38;68045:4;68041:49;68038:136;;;68093:79;;:::i;:::-;68038:136;67547:634;67441:740;;;;;:::o;68187:170::-;68327:22;68323:1;68315:6;68311:14;68304:46;68187:170;:::o;68363:366::-;68505:3;68526:67;68590:2;68585:3;68526:67;:::i;:::-;68519:74;;68602:93;68691:3;68602:93;:::i;:::-;68720:2;68715:3;68711:12;68704:19;;68363:366;;;:::o;68735:419::-;68901:4;68939:2;68928:9;68924:18;68916:26;;68988:9;68982:4;68978:20;68974:1;68963:9;68959:17;68952:47;69016:131;69142:4;69016:131;:::i;:::-;69008:139;;68735:419;;;:::o;69160:553::-;69337:4;69375:3;69364:9;69360:19;69352:27;;69389:71;69457:1;69446:9;69442:17;69433:6;69389:71;:::i;:::-;69470:72;69538:2;69527:9;69523:18;69514:6;69470:72;:::i;:::-;69552;69620:2;69609:9;69605:18;69596:6;69552:72;:::i;:::-;69634;69702:2;69691:9;69687:18;69678:6;69634:72;:::i;:::-;69160:553;;;;;;;:::o;69719:85::-;69764:7;69793:5;69782:16;;69719:85;;;:::o;69810:156::-;69867:9;69900:60;69917:42;69926:32;69952:5;69926:32;:::i;:::-;69917:42;:::i;:::-;69900:60;:::i;:::-;69887:73;;69810:156;;;:::o;69972:145::-;70066:44;70104:5;70066:44;:::i;:::-;70061:3;70054:57;69972:145;;:::o;70123:236::-;70223:4;70261:2;70250:9;70246:18;70238:26;;70274:78;70349:1;70338:9;70334:17;70325:6;70274:78;:::i;:::-;70123:236;;;;:::o;70365:174::-;70505:26;70501:1;70493:6;70489:14;70482:50;70365:174;:::o;70545:366::-;70687:3;70708:67;70772:2;70767:3;70708:67;:::i;:::-;70701:74;;70784:93;70873:3;70784:93;:::i;:::-;70902:2;70897:3;70893:12;70886:19;;70545:366;;;:::o;70917:419::-;71083:4;71121:2;71110:9;71106:18;71098:26;;71170:9;71164:4;71160:20;71156:1;71145:9;71141:17;71134:47;71198:131;71324:4;71198:131;:::i;:::-;71190:139;;70917:419;;;:::o;71342:171::-;71381:3;71404:24;71422:5;71404:24;:::i;:::-;71395:33;;71450:4;71443:5;71440:15;71437:41;;71458:18;;:::i;:::-;71437:41;71505:1;71498:5;71494:13;71487:20;;71342:171;;;:::o;71519:185::-;71559:1;71576:20;71594:1;71576:20;:::i;:::-;71571:25;;71610:20;71628:1;71610:20;:::i;:::-;71605:25;;71649:1;71639:35;;71654:18;;:::i;:::-;71639:35;71696:1;71693;71689:9;71684:14;;71519:185;;;;:::o;71710:170::-;71850:22;71846:1;71838:6;71834:14;71827:46;71710:170;:::o;71886:366::-;72028:3;72049:67;72113:2;72108:3;72049:67;:::i;:::-;72042:74;;72125:93;72214:3;72125:93;:::i;:::-;72243:2;72238:3;72234:12;72227:19;;71886:366;;;:::o;72258:419::-;72424:4;72462:2;72451:9;72447:18;72439:26;;72511:9;72505:4;72501:20;72497:1;72486:9;72482:17;72475:47;72539:131;72665:4;72539:131;:::i;:::-;72531:139;;72258:419;;;:::o;72683:114::-;;:::o;72803:398::-;72962:3;72983:83;73064:1;73059:3;72983:83;:::i;:::-;72976:90;;73075:93;73164:3;73075:93;:::i;:::-;73193:1;73188:3;73184:11;73177:18;;72803:398;;;:::o;73207:379::-;73391:3;73413:147;73556:3;73413:147;:::i;:::-;73406:154;;73577:3;73570:10;;73207:379;;;:::o;73592:161::-;73732:13;73728:1;73720:6;73716:14;73709:37;73592:161;:::o;73759:366::-;73901:3;73922:67;73986:2;73981:3;73922:67;:::i;:::-;73915:74;;73998:93;74087:3;73998:93;:::i;:::-;74116:2;74111:3;74107:12;74100:19;;73759:366;;;:::o;74131:419::-;74297:4;74335:2;74324:9;74320:18;74312:26;;74384:9;74378:4;74374:20;74370:1;74359:9;74355:17;74348:47;74412:131;74538:4;74412:131;:::i;:::-;74404:139;;74131:419;;;:::o;74556:179::-;74696:31;74692:1;74684:6;74680:14;74673:55;74556:179;:::o;74741:366::-;74883:3;74904:67;74968:2;74963:3;74904:67;:::i;:::-;74897:74;;74980:93;75069:3;74980:93;:::i;:::-;75098:2;75093:3;75089:12;75082:19;;74741:366;;;:::o;75113:529::-;75307:4;75345:2;75334:9;75330:18;75322:26;;75358:71;75426:1;75415:9;75411:17;75402:6;75358:71;:::i;:::-;75476:9;75470:4;75466:20;75461:2;75450:9;75446:18;75439:48;75504:131;75630:4;75504:131;:::i;:::-;75496:139;;75113:529;;;;:::o;75648:177::-;75788:29;75784:1;75776:6;75772:14;75765:53;75648:177;:::o;75831:366::-;75973:3;75994:67;76058:2;76053:3;75994:67;:::i;:::-;75987:74;;76070:93;76159:3;76070:93;:::i;:::-;76188:2;76183:3;76179:12;76172:19;;75831:366;;;:::o;76203:529::-;76397:4;76435:2;76424:9;76420:18;76412:26;;76448:71;76516:1;76505:9;76501:17;76492:6;76448:71;:::i;:::-;76566:9;76560:4;76556:20;76551:2;76540:9;76536:18;76529:48;76594:131;76720:4;76594:131;:::i;:::-;76586:139;;76203:529;;;;:::o;76738:182::-;76878:34;76874:1;76866:6;76862:14;76855:58;76738:182;:::o;76926:366::-;77068:3;77089:67;77153:2;77148:3;77089:67;:::i;:::-;77082:74;;77165:93;77254:3;77165:93;:::i;:::-;77283:2;77278:3;77274:12;77267:19;;76926:366;;;:::o;77298:529::-;77492:4;77530:2;77519:9;77515:18;77507:26;;77543:71;77611:1;77600:9;77596:17;77587:6;77543:71;:::i;:::-;77661:9;77655:4;77651:20;77646:2;77635:9;77631:18;77624:48;77689:131;77815:4;77689:131;:::i;:::-;77681:139;;77298:529;;;;:::o;77833:233::-;77872:3;77895:24;77913:5;77895:24;:::i;:::-;77886:33;;77941:66;77934:5;77931:77;77928:103;;78011:18;;:::i;:::-;77928:103;78058:1;78051:5;78047:13;78040:20;;77833:233;;;:::o;78258:724::-;78335:4;78341:6;78397:11;78384:25;78497:1;78491:4;78487:12;78476:8;78460:14;78456:29;78452:48;78432:18;78428:73;78418:168;;78505:79;;:::i;:::-;78418:168;78617:18;78607:8;78603:33;78595:41;;78669:4;78656:18;78646:28;;78697:18;78689:6;78686:30;78683:117;;;78719:79;;:::i;:::-;78683:117;78827:2;78821:4;78817:13;78809:21;;78884:4;78876:6;78872:17;78856:14;78852:38;78846:4;78842:49;78839:136;;;78894:79;;:::i;:::-;78839:136;78348:634;78258:724;;;;;:::o;78988:96::-;79046:6;79074:3;79064:13;;78988:96;;;;:::o;79090:140::-;79138:4;79161:3;79153:11;;79184:3;79181:1;79174:14;79218:4;79215:1;79205:18;79197:26;;79090:140;;;:::o;79236:93::-;79273:6;79320:2;79315;79308:5;79304:14;79300:23;79290:33;;79236:93;;;:::o;79335:107::-;79379:8;79429:5;79423:4;79419:16;79398:37;;79335:107;;;;:::o;79448:393::-;79517:6;79567:1;79555:10;79551:18;79590:97;79620:66;79609:9;79590:97;:::i;:::-;79708:39;79738:8;79727:9;79708:39;:::i;:::-;79696:51;;79780:4;79776:9;79769:5;79765:21;79756:30;;79829:4;79819:8;79815:19;79808:5;79805:30;79795:40;;79524:317;;79448:393;;;;;:::o;79847:142::-;79897:9;79930:53;79948:34;79957:24;79975:5;79957:24;:::i;:::-;79948:34;:::i;:::-;79930:53;:::i;:::-;79917:66;;79847:142;;;:::o;79995:75::-;80038:3;80059:5;80052:12;;79995:75;;;:::o;80076:269::-;80186:39;80217:7;80186:39;:::i;:::-;80247:91;80296:41;80320:16;80296:41;:::i;:::-;80288:6;80281:4;80275:11;80247:91;:::i;:::-;80241:4;80234:105;80152:193;80076:269;;;:::o;80351:73::-;80396:3;80351:73;:::o;80430:189::-;80507:32;;:::i;:::-;80548:65;80606:6;80598;80592:4;80548:65;:::i;:::-;80483:136;80430:189;;:::o;80625:186::-;80685:120;80702:3;80695:5;80692:14;80685:120;;;80756:39;80793:1;80786:5;80756:39;:::i;:::-;80729:1;80722:5;80718:13;80709:22;;80685:120;;;80625:186;;:::o;80817:541::-;80917:2;80912:3;80909:11;80906:445;;;80951:37;80982:5;80951:37;:::i;:::-;81034:29;81052:10;81034:29;:::i;:::-;81024:8;81020:44;81217:2;81205:10;81202:18;81199:49;;;81238:8;81223:23;;81199:49;81261:80;81317:22;81335:3;81317:22;:::i;:::-;81307:8;81303:37;81290:11;81261:80;:::i;:::-;80921:430;;80906:445;80817:541;;;:::o;81364:117::-;81418:8;81468:5;81462:4;81458:16;81437:37;;81364:117;;;;:::o;81487:169::-;81531:6;81564:51;81612:1;81608:6;81600:5;81597:1;81593:13;81564:51;:::i;:::-;81560:56;81645:4;81639;81635:15;81625:25;;81538:118;81487:169;;;;:::o;81661:295::-;81737:4;81883:29;81908:3;81902:4;81883:29;:::i;:::-;81875:37;;81945:3;81942:1;81938:11;81932:4;81929:21;81921:29;;81661:295;;;;:::o;81961:1398::-;82083:43;82122:3;82117;82083:43;:::i;:::-;82191:18;82183:6;82180:30;82177:56;;;82213:18;;:::i;:::-;82177:56;82257:38;82289:4;82283:11;82257:38;:::i;:::-;82342:66;82401:6;82393;82387:4;82342:66;:::i;:::-;82435:1;82464:2;82456:6;82453:14;82481:1;82476:631;;;;83151:1;83168:6;83165:84;;;83224:9;83219:3;83215:19;83202:33;83193:42;;83165:84;83275:67;83335:6;83328:5;83275:67;:::i;:::-;83269:4;83262:81;83124:229;82446:907;;82476:631;82528:4;82524:9;82516:6;82512:22;82562:36;82593:4;82562:36;:::i;:::-;82620:1;82634:215;82648:7;82645:1;82642:14;82634:215;;;82734:9;82729:3;82725:19;82712:33;82704:6;82697:49;82785:1;82777:6;82773:14;82763:24;;82832:2;82821:9;82817:18;82804:31;;82671:4;82668:1;82664:12;82659:17;;82634:215;;;82877:6;82868:7;82865:19;82862:186;;;82942:9;82937:3;82933:19;82920:33;82985:48;83027:4;83019:6;83015:17;83004:9;82985:48;:::i;:::-;82977:6;82970:64;82885:163;82862:186;83094:1;83090;83082:6;83078:14;83074:22;83068:4;83061:36;82483:624;;;82446:907;;82058:1301;;;81961:1398;;;:::o;83365:214::-;83478:95;83565:7;83556;83550:4;83478:95;:::i;:::-;83365:214;;;:::o;83585:483::-;83753:1;83747:4;83743:12;83799:1;83792:5;83788:13;83864:62;83913:12;83906:5;83864:62;:::i;:::-;83940:110;84036:13;84021;84009:10;83940:110;:::i;:::-;83710:351;;;;83585:483;;:::o;84074:240::-;84202:106;84300:7;84294:4;84202:106;:::i;:::-;84074:240;;:::o;84320:247::-;84460:34;84456:1;84448:6;84444:14;84437:58;84529:30;84524:2;84516:6;84512:15;84505:55;84320:247;:::o;84573:366::-;84715:3;84736:67;84800:2;84795:3;84736:67;:::i;:::-;84729:74;;84812:93;84901:3;84812:93;:::i;:::-;84930:2;84925:3;84921:12;84914:19;;84573:366;;;:::o;84945:419::-;85111:4;85149:2;85138:9;85134:18;85126:26;;85198:9;85192:4;85188:20;85184:1;85173:9;85169:17;85162:47;85226:131;85352:4;85226:131;:::i;:::-;85218:139;;84945:419;;;:::o;85370:243::-;85510:34;85506:1;85498:6;85494:14;85487:58;85579:26;85574:2;85566:6;85562:15;85555:51;85370:243;:::o;85619:366::-;85761:3;85782:67;85846:2;85841:3;85782:67;:::i;:::-;85775:74;;85858:93;85947:3;85858:93;:::i;:::-;85976:2;85971:3;85967:12;85960:19;;85619:366;;;:::o;85991:419::-;86157:4;86195:2;86184:9;86180:18;86172:26;;86244:9;86238:4;86234:20;86230:1;86219:9;86215:17;86208:47;86272:131;86398:4;86272:131;:::i;:::-;86264:139;;85991:419;;;:::o;86416:238::-;86556:34;86552:1;86544:6;86540:14;86533:58;86625:21;86620:2;86612:6;86608:15;86601:46;86416:238;:::o;86660:366::-;86802:3;86823:67;86887:2;86882:3;86823:67;:::i;:::-;86816:74;;86899:93;86988:3;86899:93;:::i;:::-;87017:2;87012:3;87008:12;87001:19;;86660:366;;;:::o;87032:419::-;87198:4;87236:2;87225:9;87221:18;87213:26;;87285:9;87279:4;87275:20;87271:1;87260:9;87256:17;87249:47;87313:131;87439:4;87313:131;:::i;:::-;87305:139;;87032:419;;;:::o;87457:102::-;87499:8;87546:5;87543:1;87539:13;87518:34;;87457:102;;;:::o;87565:848::-;87626:5;87633:4;87657:6;87648:15;;87681:5;87672:14;;87695:712;87716:1;87706:8;87703:15;87695:712;;;87811:4;87806:3;87802:14;87796:4;87793:24;87790:50;;;87820:18;;:::i;:::-;87790:50;87870:1;87860:8;87856:16;87853:451;;;88285:4;88278:5;88274:16;88265:25;;87853:451;88335:4;88329;88325:15;88317:23;;88365:32;88388:8;88365:32;:::i;:::-;88353:44;;87695:712;;;87565:848;;;;;;;:::o;88419:1073::-;88473:5;88664:8;88654:40;;88685:1;88676:10;;88687:5;;88654:40;88713:4;88703:36;;88730:1;88721:10;;88732:5;;88703:36;88799:4;88847:1;88842:27;;;;88883:1;88878:191;;;;88792:277;;88842:27;88860:1;88851:10;;88862:5;;;88878:191;88923:3;88913:8;88910:17;88907:43;;;88930:18;;:::i;:::-;88907:43;88979:8;88976:1;88972:16;88963:25;;89014:3;89007:5;89004:14;89001:40;;;89021:18;;:::i;:::-;89001:40;89054:5;;;88792:277;;89178:2;89168:8;89165:16;89159:3;89153:4;89150:13;89146:36;89128:2;89118:8;89115:16;89110:2;89104:4;89101:12;89097:35;89081:111;89078:246;;;89234:8;89228:4;89224:19;89215:28;;89269:3;89262:5;89259:14;89256:40;;;89276:18;;:::i;:::-;89256:40;89309:5;;89078:246;89349:42;89387:3;89377:8;89371:4;89368:1;89349:42;:::i;:::-;89334:57;;;;89423:4;89418:3;89414:14;89407:5;89404:25;89401:51;;;89432:18;;:::i;:::-;89401:51;89481:4;89474:5;89470:16;89461:25;;88419:1073;;;;;;:::o;89498:93::-;89534:7;89574:10;89567:5;89563:22;89552:33;;89498:93;;;:::o;89597:283::-;89656:5;89680:23;89698:4;89680:23;:::i;:::-;89672:31;;89724:26;89741:8;89724:26;:::i;:::-;89712:38;;89769:104;89806:66;89796:8;89790:4;89769:104;:::i;:::-;89760:113;;89597:283;;;;:::o;89886:226::-;89920:3;89943:22;89959:5;89943:22;:::i;:::-;89934:31;;89987:66;89980:5;89977:77;89974:103;;90057:18;;:::i;:::-;89974:103;90100:5;90097:1;90093:13;90086:20;;89886:226;;;:::o;90118:122::-;90191:24;90209:5;90191:24;:::i;:::-;90184:5;90181:35;90171:63;;90230:1;90227;90220:12;90171:63;90118:122;:::o;90246:143::-;90303:5;90334:6;90328:13;90319:22;;90350:33;90377:5;90350:33;:::i;:::-;90246:143;;;;:::o;90395:351::-;90465:6;90514:2;90502:9;90493:7;90489:23;90485:32;90482:119;;;90520:79;;:::i;:::-;90482:119;90640:1;90665:64;90721:7;90712:6;90701:9;90697:22;90665:64;:::i;:::-;90655:74;;90611:128;90395:351;;;;:::o;90752:171::-;90892:23;90888:1;90880:6;90876:14;90869:47;90752:171;:::o;90929:366::-;91071:3;91092:67;91156:2;91151:3;91092:67;:::i;:::-;91085:74;;91168:93;91257:3;91168:93;:::i;:::-;91286:2;91281:3;91277:12;91270:19;;90929:366;;;:::o;91301:419::-;91467:4;91505:2;91494:9;91490:18;91482:26;;91554:9;91548:4;91544:20;91540:1;91529:9;91525:17;91518:47;91582:131;91708:4;91582:131;:::i;:::-;91574:139;;91301:419;;;:::o;91726:116::-;91777:4;91800:3;91792:11;;91830:4;91825:3;91821:14;91813:22;;91726:116;;;:::o;91848:154::-;91891:11;91927:29;91951:3;91945:10;91927:29;:::i;:::-;91990:5;91966:29;;91903:99;91848:154;;;:::o;92008:594::-;92092:5;92123:38;92155:5;92123:38;:::i;:::-;92186:5;92213:40;92247:5;92213:40;:::i;:::-;92201:52;;92272:35;92298:8;92272:35;:::i;:::-;92263:44;;92331:2;92323:6;92320:14;92317:278;;;92402:169;92487:66;92457:6;92453:2;92449:15;92446:1;92442:23;92402:169;:::i;:::-;92379:5;92358:227;92349:236;;92317:278;92098:504;;92008:594;;;:::o;92608:430::-;92751:4;92789:2;92778:9;92774:18;92766:26;;92802:71;92870:1;92859:9;92855:17;92846:6;92802:71;:::i;:::-;92883:70;92949:2;92938:9;92934:18;92925:6;92883:70;:::i;:::-;92963:68;93027:2;93016:9;93012:18;93003:6;92963:68;:::i;:::-;92608:430;;;;;;:::o;93044:221::-;93184:34;93180:1;93172:6;93168:14;93161:58;93253:4;93248:2;93240:6;93236:15;93229:29;93044:221;:::o;93271:366::-;93413:3;93434:67;93498:2;93493:3;93434:67;:::i;:::-;93427:74;;93510:93;93599:3;93510:93;:::i;:::-;93628:2;93623:3;93619:12;93612:19;;93271:366;;;:::o;93643:419::-;93809:4;93847:2;93836:9;93832:18;93824:26;;93896:9;93890:4;93886:20;93882:1;93871:9;93867:17;93860:47;93924:131;94050:4;93924:131;:::i;:::-;93916:139;;93643:419;;;:::o;94068:76::-;94104:7;94133:5;94122:16;;94068:76;;;:::o;94150:228::-;94185:3;94208:23;94225:5;94208:23;:::i;:::-;94199:32;;94253:66;94246:5;94243:77;94240:103;;94323:18;;:::i;:::-;94240:103;94366:5;94363:1;94359:13;94352:20;;94150:228;;;:::o","linkReferences":{},"immutableReferences":{"468":[{"start":15849,"length":32},{"start":15934,"length":32},{"start":16376,"length":32}]}},"methodIdentifiers":{"BURN_ACTOR()":"0a6a63f1","EXTRA_DATA_MAX_SIZE()":"029b4646","FIL_USD_PRICE_FEED_ID()":"19c75950","LEAF_SIZE()":"c0e15949","MAX_ENQUEUED_REMOVALS()":"9f8cb3bd","MAX_ROOT_SIZE()":"16e2bcd5","NO_CHALLENGE_SCHEDULED()":"462dd449","NO_PROVEN_EPOCH()":"f178b1be","PYTH()":"67e406d5","RANDOMNESS_PRECOMPILE()":"15b17570","SECONDS_IN_DAY()":"61a52a36","UPGRADE_INTERFACE_VERSION()":"ad3cb1cc","addRoots(uint256,((bytes),uint256)[],bytes)":"11c0ee4a","calculateProofFee(uint256,uint256)":"4903704a","claimProofSetOwnership(uint256)":"ee3dac65","createProofSet(address,bytes)":"0a4d7932","deleteProofSet(uint256,bytes)":"847d1d06","findRootIds(uint256,uint256[])":"0528a55b","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getFILUSDPrice()":"4fa27920","getNextChallengeEpoch(uint256)":"6ba4608f","getNextProofSetId()":"8ea417e5","getNextRootId(uint256)":"d49245c1","getProofSetLastProvenEpoch(uint256)":"faa67163","getProofSetLeafCount(uint256)":"3f84135f","getProofSetListener(uint256)":"31601226","getProofSetOwner(uint256)":"4726075b","getRandomness(uint256)":"453f4f62","getRootCid(uint256,uint256)":"3b7ae913","getRootLeafCount(uint256,uint256)":"9153e64b","getScheduledRemovals(uint256)":"6fa44692","initialize(uint256)":"fe4b84df","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","owner()":"8da5cb5b","proofSetLive(uint256)":"f5cac1ba","proposeProofSetOwner(uint256,address)":"6cb55c16","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","proxiableUUID()":"52d1902d","renounceOwnership()":"715018a6","rootChallengable(uint256,uint256)":"71cf2a16","rootLive(uint256,uint256)":"47331050","scheduleRemovals(uint256,uint256[],bytes)":"3b68e4e9","transferOwnership(address)":"f2fde38b","upgradeToAndCall(address,bytes)":"4f1ef286"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"target\",\"type\":\"address\"}],\"name\":\"AddressEmptyCode\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"ERC1967InvalidImplementation\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"ERC1967NonPayable\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"FailedCall\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"idx\",\"type\":\"uint256\"},{\"internalType\":\"string\",\"name\":\"msg\",\"type\":\"string\"}],\"name\":\"IndexedError\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"InvalidInitialization\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"NotInitializing\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"OwnableInvalidOwner\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"account\",\"type\":\"address\"}],\"name\":\"OwnableUnauthorizedAccount\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"UUPSUnauthorizedCallContext\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"slot\",\"type\":\"bytes32\"}],\"name\":\"UUPSUnsupportedProxiableUUID\",\"type\":\"error\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"message\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"value\",\"type\":\"uint256\"}],\"name\":\"Debug\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"version\",\"type\":\"uint64\"}],\"name\":\"Initialized\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"previousOwner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"OwnershipTransferred\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct PDPVerifier.RootIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"ProofSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"ProofSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"ProofSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldOwner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"ProofSetOwnerChanged\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"rootIds\",\"type\":\"uint256[]\"}],\"name\":\"RootsAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"rootIds\",\"type\":\"uint256[]\"}],\"name\":\"RootsRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"Upgraded\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"BURN_ACTOR\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"EXTRA_DATA_MAX_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"FIL_USD_PRICE_FEED_ID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"LEAF_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_ENQUEUED_REMOVALS\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_ROOT_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_CHALLENGE_SCHEDULED\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_PROVEN_EPOCH\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PYTH\",\"outputs\":[{\"internalType\":\"contract IPyth\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"RANDOMNESS_PRECOMPILE\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"SECONDS_IN_DAY\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"UPGRADE_INTERFACE_VERSION\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"root\",\"type\":\"tuple\"},{\"internalType\":\"uint256\",\"name\":\"rawSize\",\"type\":\"uint256\"}],\"internalType\":\"struct PDPVerifier.RootData[]\",\"name\":\"rootData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addRoots\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"estimatedGasFee\",\"type\":\"uint256\"}],\"name\":\"calculateProofFee\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"claimProofSetOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createProofSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteProofSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findRootIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct PDPVerifier.RootIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getFILUSDPrice\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"},{\"internalType\":\"int32\",\"name\":\"\",\"type\":\"int32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextProofSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextRootId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getProofSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getProofSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getProofSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getProofSetOwner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"epoch\",\"type\":\"uint256\"}],\"name\":\"getRandomness\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"}],\"name\":\"getRootCid\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"\",\"type\":\"tuple\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"}],\"name\":\"getRootLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"_challengeFinality\",\"type\":\"uint256\"}],\"name\":\"initialize\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"owner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"proofSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"proposeProofSetOwner\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct PDPVerifier.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"proxiableUUID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"renounceOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"}],\"name\":\"rootChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"rootId\",\"type\":\"uint256\"}],\"name\":\"rootLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"rootIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"scheduleRemovals\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"transferOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"name\":\"upgradeToAndCall\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"}],\"devdoc\":{\"errors\":{\"AddressEmptyCode(address)\":[{\"details\":\"There's no code at `target` (it is not a contract).\"}],\"ERC1967InvalidImplementation(address)\":[{\"details\":\"The `implementation` of the proxy is invalid.\"}],\"ERC1967NonPayable()\":[{\"details\":\"An upgrade function sees `msg.value > 0` that may be lost.\"}],\"FailedCall()\":[{\"details\":\"A call to an address target failed. The target may have reverted.\"}],\"InvalidInitialization()\":[{\"details\":\"The contract is already initialized.\"}],\"NotInitializing()\":[{\"details\":\"The contract is not initializing.\"}],\"OwnableInvalidOwner(address)\":[{\"details\":\"The owner is not a valid owner account. (eg. `address(0)`)\"}],\"OwnableUnauthorizedAccount(address)\":[{\"details\":\"The caller account is not authorized to perform an operation.\"}],\"UUPSUnauthorizedCallContext()\":[{\"details\":\"The call is from an unauthorized context.\"}],\"UUPSUnsupportedProxiableUUID(bytes32)\":[{\"details\":\"The storage `slot` is unsupported as a UUID.\"}]},\"events\":{\"Initialized(uint64)\":{\"details\":\"Triggered when the contract has been initialized or reinitialized.\"},\"Upgraded(address)\":{\"details\":\"Emitted when the implementation is upgraded.\"}},\"kind\":\"dev\",\"methods\":{\"constructor\":{\"custom:oz-upgrades-unsafe-allow\":\"constructor\"},\"owner()\":{\"details\":\"Returns the address of the current owner.\"},\"proxiableUUID()\":{\"details\":\"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier.\"},\"renounceOwnership()\":{\"details\":\"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner.\"},\"transferOwnership(address)\":{\"details\":\"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner.\"},\"upgradeToAndCall(address,bytes)\":{\"custom:oz-upgrades-unsafe-allow-reachable\":\"delegatecall\",\"details\":\"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event.\"}},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/PDPVerifier.sol\":\"PDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":false,\"runs\":200},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=node_modules/@pythnetwork/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\"]},\"sources\":{\"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol\":{\"keccak256\":\"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6\",\"dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol\":{\"keccak256\":\"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609\",\"dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol\":{\"keccak256\":\"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0\",\"dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol\":{\"keccak256\":\"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9\",\"dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV\"]},\"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol\":{\"keccak256\":\"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d\",\"dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7\"]},\"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol\":{\"keccak256\":\"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196\",\"dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA\"]},\"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol\":{\"keccak256\":\"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a\",\"dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw\"]},\"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol\":{\"keccak256\":\"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa\",\"dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM\"]},\"lib/openzeppelin-contracts/contracts/utils/Address.sol\":{\"keccak256\":\"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23\",\"dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb\"]},\"lib/openzeppelin-contracts/contracts/utils/Errors.sol\":{\"keccak256\":\"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf\",\"dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB\"]},\"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol\":{\"keccak256\":\"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b\",\"dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM\"]},\"node_modules/@pythnetwork/pyth-sdk-solidity/IPyth.sol\":{\"keccak256\":\"0x217532ece69b8e472a6260b740c34aebfb5a299bbfed6392cf0458ed368be7ab\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://02d1b71006ccdfd6402a2b72ea197babbd1b54c26a70ebb76a114f0ae8352f08\",\"dweb:/ipfs/QmbqfuvwriG3AEwYEwupUaQKgfxRYK6Qui99o6wQysPoP3\"]},\"node_modules/@pythnetwork/pyth-sdk-solidity/IPythEvents.sol\":{\"keccak256\":\"0x7ca8e03315d4516d6833c425a52c43e8cacf2077492074d2d36ae5c17899c9c8\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://ad1c69d157eccb09ce248e1ec021f2e58b61dd36160f5be3973a7bea4a899f64\",\"dweb:/ipfs/QmW1yXsDrMsuQKxtZanSZXpyUW2QwnCKVoCjS5fC3NoSVY\"]},\"node_modules/@pythnetwork/pyth-sdk-solidity/PythStructs.sol\":{\"keccak256\":\"0xade221177dda98ebd194c363f264ceea125bde0e6a7a72f7b54da3ac60316894\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://a404dbbc64183995326c345cae27601d37c783b3d9030c8dc0ab4943fa2bf1da\",\"dweb:/ipfs/QmfNFesQffYisafmJFbKHxVFSD8fY49X1z9f8N7qtfW8AX\"]},\"src/BitOps.sol\":{\"keccak256\":\"0x55fc8272df01302eba6fde6174e691ec86f791c39ac9b1c6a5e4ca1792439ca4\",\"license\":\"UNLICENSED\",\"urls\":[\"bzz-raw://1e4de6ed5f6e6180261728a590eeb629de65db443f4f279801c03a1bc14201d7\",\"dweb:/ipfs/QmeCcCjy88QJwCkZoGbeZVjxksePwTcmhKevtA2F3kRXaT\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4085c3a55cdf809251a469829bae218d03db4afd9455dab674a8a2ab3b7451dc\",\"license\":\"UNLICENSED\",\"urls\":[\"bzz-raw://93406cf5db1b0fa908c306ab6c4d42f69990e9fd08c781de871724c525097803\",\"dweb:/ipfs/QmXkXwjhEo929M6qBXkHKBT3DowiVYcLEe5oUkFnjFJMy2\"]},\"src/Fees.sol\":{\"keccak256\":\"0x74945bddcdd334715c9fab53deba13867f17855976ae64c33abdc84dc439feb0\",\"license\":\"UNLICENSED\",\"urls\":[\"bzz-raw://8c7ee756cc406b4192220c54c3e66908a32887c32519461e2dbae8eff144691c\",\"dweb:/ipfs/Qmdy23cwyFG2xbv1htf8FCGwFPfY98rfoKFJ9G6zcWiRkk\"]},\"src/PDPVerifier.sol\":{\"keccak256\":\"0x281e77ce7e1f0eef0d3db9be6116ed71cee92c3b8181dc3c47ff582878ca7fb7\",\"license\":\"UNLICENSED\",\"urls\":[\"bzz-raw://63b491e882199f57ce1c5f06ce57b257e40d9c6d38eb62be61d9eba4c7103fe3\",\"dweb:/ipfs/QmQr7NGETKtAcbHJhQ6ZxwB5yoHAwukjVjZwQQGd836KRk\"]},\"src/Proofs.sol\":{\"keccak256\":\"0xf8d27dd91086ba2b4521f36227d92aae35c9f8dfcb117c775e2417166e15a737\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8db46f122470a14d2c084655c6fada18c966accca05feae92923b1ace7a9f86b\",\"dweb:/ipfs/QmQWGxWUcpejzJt28gwbKfq5C3LLiB5HrHdXMja6HHYxbj\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"inputs":[{"internalType":"address","name":"target","type":"address"}],"type":"error","name":"AddressEmptyCode"},{"inputs":[{"internalType":"address","name":"implementation","type":"address"}],"type":"error","name":"ERC1967InvalidImplementation"},{"inputs":[],"type":"error","name":"ERC1967NonPayable"},{"inputs":[],"type":"error","name":"FailedCall"},{"inputs":[{"internalType":"uint256","name":"idx","type":"uint256"},{"internalType":"string","name":"msg","type":"string"}],"type":"error","name":"IndexedError"},{"inputs":[],"type":"error","name":"InvalidInitialization"},{"inputs":[],"type":"error","name":"NotInitializing"},{"inputs":[{"internalType":"address","name":"owner","type":"address"}],"type":"error","name":"OwnableInvalidOwner"},{"inputs":[{"internalType":"address","name":"account","type":"address"}],"type":"error","name":"OwnableUnauthorizedAccount"},{"inputs":[],"type":"error","name":"UUPSUnauthorizedCallContext"},{"inputs":[{"internalType":"bytes32","name":"slot","type":"bytes32"}],"type":"error","name":"UUPSUnsupportedProxiableUUID"},{"inputs":[{"internalType":"string","name":"message","type":"string","indexed":false},{"internalType":"uint256","name":"value","type":"uint256","indexed":false}],"type":"event","name":"Debug","anonymous":false},{"inputs":[{"internalType":"uint64","name":"version","type":"uint64","indexed":false}],"type":"event","name":"Initialized","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"address","name":"previousOwner","type":"address","indexed":true},{"internalType":"address","name":"newOwner","type":"address","indexed":true}],"type":"event","name":"OwnershipTransferred","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct PDPVerifier.RootIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"rootId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"owner","type":"address","indexed":true}],"type":"event","name":"ProofSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"ProofSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"ProofSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldOwner","type":"address","indexed":true},{"internalType":"address","name":"newOwner","type":"address","indexed":true}],"type":"event","name":"ProofSetOwnerChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"rootIds","type":"uint256[]","indexed":false}],"type":"event","name":"RootsAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"rootIds","type":"uint256[]","indexed":false}],"type":"event","name":"RootsRemoved","anonymous":false},{"inputs":[{"internalType":"address","name":"implementation","type":"address","indexed":true}],"type":"event","name":"Upgraded","anonymous":false},{"inputs":[],"stateMutability":"view","type":"function","name":"BURN_ACTOR","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"EXTRA_DATA_MAX_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"FIL_USD_PRICE_FEED_ID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"LEAF_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_ENQUEUED_REMOVALS","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_ROOT_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_CHALLENGE_SCHEDULED","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_PROVEN_EPOCH","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"PYTH","outputs":[{"internalType":"contract IPyth","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"RANDOMNESS_PRECOMPILE","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"SECONDS_IN_DAY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"UPGRADE_INTERFACE_VERSION","outputs":[{"internalType":"string","name":"","type":"string"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct PDPVerifier.RootData[]","name":"rootData","type":"tuple[]","components":[{"internalType":"struct Cids.Cid","name":"root","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"uint256","name":"rawSize","type":"uint256"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addRoots","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"estimatedGasFee","type":"uint256"}],"stateMutability":"view","type":"function","name":"calculateProofFee","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"claimProofSetOwnership"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createProofSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteProofSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findRootIds","outputs":[{"internalType":"struct PDPVerifier.RootIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"rootId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getFILUSDPrice","outputs":[{"internalType":"uint64","name":"","type":"uint64"},{"internalType":"int32","name":"","type":"int32"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextProofSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextRootId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getProofSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getProofSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getProofSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getProofSetOwner","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"epoch","type":"uint256"}],"stateMutability":"view","type":"function","name":"getRandomness","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"rootId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getRootCid","outputs":[{"internalType":"struct Cids.Cid","name":"","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"rootId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getRootLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"_challengeFinality","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"initialize"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[],"stateMutability":"view","type":"function","name":"owner","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"proofSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newOwner","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeProofSetOwner"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct PDPVerifier.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[],"stateMutability":"view","type":"function","name":"proxiableUUID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"renounceOwnership"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"rootId","type":"uint256"}],"stateMutability":"view","type":"function","name":"rootChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"rootId","type":"uint256"}],"stateMutability":"view","type":"function","name":"rootLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"rootIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"scheduleRemovals"},{"inputs":[{"internalType":"address","name":"newOwner","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"transferOwnership"},{"inputs":[{"internalType":"address","name":"newImplementation","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"stateMutability":"payable","type":"function","name":"upgradeToAndCall"}],"devdoc":{"kind":"dev","methods":{"constructor":{"custom:oz-upgrades-unsafe-allow":"constructor"},"owner()":{"details":"Returns the address of the current owner."},"proxiableUUID()":{"details":"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier."},"renounceOwnership()":{"details":"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner."},"transferOwnership(address)":{"details":"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner."},"upgradeToAndCall(address,bytes)":{"custom:oz-upgrades-unsafe-allow-reachable":"delegatecall","details":"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event."}},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=node_modules/@pythnetwork/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/"],"optimizer":{"enabled":false,"runs":200},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/PDPVerifier.sol":"PDPVerifier"},"evmVersion":"shanghai","libraries":{}},"sources":{"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol":{"keccak256":"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a","urls":["bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6","dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol":{"keccak256":"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b","urls":["bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609","dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol":{"keccak256":"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a","urls":["bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0","dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol":{"keccak256":"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397","urls":["bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9","dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol":{"keccak256":"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486","urls":["bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d","dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol":{"keccak256":"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d","urls":["bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196","dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol":{"keccak256":"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128","urls":["bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a","dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol":{"keccak256":"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c","urls":["bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa","dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Address.sol":{"keccak256":"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5","urls":["bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23","dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Errors.sol":{"keccak256":"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123","urls":["bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf","dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol":{"keccak256":"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97","urls":["bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b","dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM"],"license":"MIT"},"node_modules/@pythnetwork/pyth-sdk-solidity/IPyth.sol":{"keccak256":"0x217532ece69b8e472a6260b740c34aebfb5a299bbfed6392cf0458ed368be7ab","urls":["bzz-raw://02d1b71006ccdfd6402a2b72ea197babbd1b54c26a70ebb76a114f0ae8352f08","dweb:/ipfs/QmbqfuvwriG3AEwYEwupUaQKgfxRYK6Qui99o6wQysPoP3"],"license":"Apache-2.0"},"node_modules/@pythnetwork/pyth-sdk-solidity/IPythEvents.sol":{"keccak256":"0x7ca8e03315d4516d6833c425a52c43e8cacf2077492074d2d36ae5c17899c9c8","urls":["bzz-raw://ad1c69d157eccb09ce248e1ec021f2e58b61dd36160f5be3973a7bea4a899f64","dweb:/ipfs/QmW1yXsDrMsuQKxtZanSZXpyUW2QwnCKVoCjS5fC3NoSVY"],"license":"Apache-2.0"},"node_modules/@pythnetwork/pyth-sdk-solidity/PythStructs.sol":{"keccak256":"0xade221177dda98ebd194c363f264ceea125bde0e6a7a72f7b54da3ac60316894","urls":["bzz-raw://a404dbbc64183995326c345cae27601d37c783b3d9030c8dc0ab4943fa2bf1da","dweb:/ipfs/QmfNFesQffYisafmJFbKHxVFSD8fY49X1z9f8N7qtfW8AX"],"license":"Apache-2.0"},"src/BitOps.sol":{"keccak256":"0x55fc8272df01302eba6fde6174e691ec86f791c39ac9b1c6a5e4ca1792439ca4","urls":["bzz-raw://1e4de6ed5f6e6180261728a590eeb629de65db443f4f279801c03a1bc14201d7","dweb:/ipfs/QmeCcCjy88QJwCkZoGbeZVjxksePwTcmhKevtA2F3kRXaT"],"license":"UNLICENSED"},"src/Cids.sol":{"keccak256":"0x4085c3a55cdf809251a469829bae218d03db4afd9455dab674a8a2ab3b7451dc","urls":["bzz-raw://93406cf5db1b0fa908c306ab6c4d42f69990e9fd08c781de871724c525097803","dweb:/ipfs/QmXkXwjhEo929M6qBXkHKBT3DowiVYcLEe5oUkFnjFJMy2"],"license":"UNLICENSED"},"src/Fees.sol":{"keccak256":"0x74945bddcdd334715c9fab53deba13867f17855976ae64c33abdc84dc439feb0","urls":["bzz-raw://8c7ee756cc406b4192220c54c3e66908a32887c32519461e2dbae8eff144691c","dweb:/ipfs/Qmdy23cwyFG2xbv1htf8FCGwFPfY98rfoKFJ9G6zcWiRkk"],"license":"UNLICENSED"},"src/PDPVerifier.sol":{"keccak256":"0x281e77ce7e1f0eef0d3db9be6116ed71cee92c3b8181dc3c47ff582878ca7fb7","urls":["bzz-raw://63b491e882199f57ce1c5f06ce57b257e40d9c6d38eb62be61d9eba4c7103fe3","dweb:/ipfs/QmQr7NGETKtAcbHJhQ6ZxwB5yoHAwukjVjZwQQGd836KRk"],"license":"UNLICENSED"},"src/Proofs.sol":{"keccak256":"0xf8d27dd91086ba2b4521f36227d92aae35c9f8dfcb117c775e2417166e15a737","urls":["bzz-raw://8db46f122470a14d2c084655c6fada18c966accca05feae92923b1ace7a9f86b","dweb:/ipfs/QmQWGxWUcpejzJt28gwbKfq5C3LLiB5HrHdXMja6HHYxbj"],"license":"MIT"}},"version":1},"id":17} \ No newline at end of file +{"abi":[{"type":"constructor","inputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"BURN_ACTOR","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"EXTRA_DATA_MAX_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"FIL_USD_PRICE_FEED_ID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"LEAF_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_ENQUEUED_REMOVALS","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_PIECE_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_CHALLENGE_SCHEDULED","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_PROVEN_EPOCH","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"PYTH","inputs":[],"outputs":[{"name":"","type":"address","internalType":"contract IPyth"}],"stateMutability":"view"},{"type":"function","name":"RANDOMNESS_PRECOMPILE","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"SECONDS_IN_DAY","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"UPGRADE_INTERFACE_VERSION","inputs":[],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"},{"type":"function","name":"VERSION","inputs":[],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"},{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct IPDPTypes.PieceData[]","components":[{"name":"piece","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"rawSize","type":"uint256","internalType":"uint256"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"calculateProofFee","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"estimatedGasFee","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getActivePieceCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"activeCount","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getActivePieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"},{"name":"limit","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"pieces","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"rawSizes","type":"uint256[]","internalType":"uint256[]"},{"name":"hasMore","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getFILUSDPrice","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"},{"name":"","type":"int32","internalType":"int32"}],"stateMutability":"nonpayable"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getRandomness","inputs":[{"name":"epoch","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"initialize","inputs":[{"name":"_challengeFinality","type":"uint256","internalType":"uint256"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"migrate","inputs":[],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"owner","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"proxiableUUID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"renounceOwnership","inputs":[],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"transferOwnership","inputs":[{"name":"newOwner","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"upgradeToAndCall","inputs":[{"name":"newImplementation","type":"address","internalType":"address"},{"name":"data","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"payable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"implementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"Initialized","inputs":[{"name":"version","type":"uint64","indexed":false,"internalType":"uint64"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"OwnershipTransferred","inputs":[{"name":"previousOwner","type":"address","indexed":true,"internalType":"address"},{"name":"newOwner","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"reason","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"Upgraded","inputs":[{"name":"implementation","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"error","name":"AddressEmptyCode","inputs":[{"name":"target","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967InvalidImplementation","inputs":[{"name":"implementation","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967NonPayable","inputs":[]},{"type":"error","name":"FailedCall","inputs":[]},{"type":"error","name":"IndexedError","inputs":[{"name":"idx","type":"uint256","internalType":"uint256"},{"name":"msg","type":"string","internalType":"string"}]},{"type":"error","name":"InvalidInitialization","inputs":[]},{"type":"error","name":"NotInitializing","inputs":[]},{"type":"error","name":"OwnableInvalidOwner","inputs":[{"name":"owner","type":"address","internalType":"address"}]},{"type":"error","name":"OwnableUnauthorizedAccount","inputs":[{"name":"account","type":"address","internalType":"address"}]},{"type":"error","name":"UUPSUnauthorizedCallContext","inputs":[]},{"type":"error","name":"UUPSUnsupportedProxiableUUID","inputs":[{"name":"slot","type":"bytes32","internalType":"bytes32"}]}],"bytecode":{"object":"0x60a08060405234620000d157306080527ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a009081549060ff8260401c16620000c257506001600160401b036002600160401b0319828216016200007c575b604051615d359081620000d682396080518181816129710152612ae50152f35b6001600160401b031990911681179091556040519081527fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d290602090a15f80806200005c565b63f92ee8a960e01b8152600490fd5b5f80fdfe610120806040526004361015610013575f80fd5b5f905f3560e01c908163029b464614613ee55750806304595c1a14613e915780630a6a63f114613e455780630c29202414613ac65780630cd7b88014613a8d57806315b1757014613a4057806319c75950146139e75780631a271225146139cd5780631c5ae80f1461397957806321b7cd1c146138ff57806325bbbedf146138c95780632b3129bb1461385f578063349c9179146137a957806339f51544146136b2578063431860801461351c578063442cded3146134d6578063453f4f621461349957806345c0b92d14612da5578063462dd449146110655780634903704a14612d305780634f1ef28614612a3d5780634fa27920146129eb57806352d1902d1461292b5780635353bdfd1461288e57806361a52a361461285257806367e406d5146128055780636ba4608f146127b15780636fa4469214612717578063715018a6146126595780637a1e29901461240057806389208ba9146123ac5780638a405abc1461236c5780638da5cb5b146122fb5780638fd3ab80146121a35780639f8cb3bd14612168578063a531998c14612114578063ad3cb1cc14612098578063bbae41cb14611db0578063c0e1594914611d77578063ca759f2714611d30578063dc63526614611b31578063ddea76cc14611277578063df0f32481461106a578063f178b1be14611065578063f2fde38b1461101a578063f58f952b1461047b578063f83758fe14610440578063fe4b84df1461028e5763ffa1ad741461023a575f80fd5b3461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876102736142b4565b60405191829160208352602083019061401e565b0390f35b80fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b577ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c16159067ffffffffffffffff811680159081610438575b600114908161042e575b159081610425575b506103fb578160017fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000083161784556103c6575b5061034c61583c565b61035461583c565b61035d33614fc6565b61036561583c565b6004358355610372575080f35b7fffffffffffffffffffffffffffffffffffffffffffffff00ffffffffffffffff81541690557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160018152a180f35b7fffffffffffffffffffffffffffffffffffffffffffffff00000000000000000016680100000000000000011782555f610343565b60046040517ff92ee8a9000000000000000000000000000000000000000000000000000000008152fd5b9050155f610310565b303b159150610308565b8391506102fe565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760209054604051908152f35b506104853661407a565b9291905a93828452600b60205273ffffffffffffffffffffffffffffffffffffffff92836040862054163303610f96578115610f385780855260076020526040852054804310610eda5715610e7c576104dd82614625565b9381865260076020526104f36040872054614b55565b8287526009602052604087205490600560205261051360408920546150db565b9261010084810311610e4f579193888888888d9484985b8267ffffffffffffffff8b161015610ac6576040518860208201528260408201527fffffffffffffffff0000000000000000000000000000000000000000000000008b60c01b1660608201526048815280608081011067ffffffffffffffff608083011117610a995760808101604052602081519101208b15610a6c576105b9908c83610100039106846152ac565b6105cd67ffffffffffffffff8c16876146a0565b526105e267ffffffffffffffff8b16866146a0565b506106026105fa67ffffffffffffffff8c16876146a0565b515183614503565b602081515110610a0e5760405190610619826141cf565b6020825260203681840137875b602081106109455750508660c0526020815191015160c0526020811061090f575b5081865260036020526040862061066867ffffffffffffffff8c16876146a0565b5151875260205260408620547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff81019081116108e2576106a7906150db565b610100039a6101008c116108e25760018c018c116108e2578a9b8b602061070567ffffffffffffffff6106fc816106ee6106e58d8d848a1691614f86565b86810190615087565b906101005295168a8c614f86565b359f168a6146a0565b5101516080526107148161460d565b60e05260405160a05261072b60e05160a0516141eb565b60a051508060a05152602060a05101368260051b6101005101116108de5761010051905b8260051b610100510182106108ce5750505060a051510361084a57608051969a96988b975b60a051518d10156107bd5760019061078e8e60a0516146a0565b51908c83166107ae57906107a191615cc4565b9a5b811c9c019b99610774565b6107b791615cc4565b9a6107a3565b9195995093979b91959992969a5060c051036107ec576107dc90614ec6565b989490979399959196929961052a565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f70726f6f6620646964206e6f74207665726966790000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f70726f6f66206c656e67746820646f6573206e6f74206d61746368207472656560448201527f20686569676874000000000000000000000000000000000000000000000000006064820152fd5b813581526020918201910161074f565b8980fd5b6024877f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9060200360031b1b60c0511660c0528b610647565b81518051807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101116109e1576109cc83926109c66001957fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe07fff0000000000000000000000000000000000000000000000000000000000000095016143c5565b9061582b565b51168a1a6109da828661582b565b5301610626565b60248b7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601560248201527f436964206461746120697320746f6f2073686f727400000000000000000000006044820152fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b5093959091610ad788965a90614600565b879188905b858210610e0057505060208201809211610dd35761051491828102928184041490151715610dd357610b1891610b11916143c5565b4890614792565b84875260096020526040872054907f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82168203610dd357610b83610b799493926040928a610b64614d26565b988196838d8c9552600d602052205443614600565b9360051b926154c3565b803410610d75575f8080808473ff000000000000000000000000000000000000635af1610bae614b26565b5015610d1757867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b6060610c03948c9867ffffffffffffffff6040519287845216602083015260030b6040820152a234614600565b9585845260086020526040842054169081610c84575b5050507f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e509183610c5b9252600d602052436040872055604051918291826140c8565b0390a28181610c675780f35b808080610c7f94335af1610c79614b26565b50614edf565b818180f35b6006602052604084205491803b15610d135784928360849260405196879586947f356de02b0000000000000000000000000000000000000000000000000000000086528c60048701526024860152604485015260648401525af18015610d0857610cf0575b8080610c19565b610cf99061419f565b610d04578385610ce9565b8380fd5b6040513d84823e3d90fd5b8480fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f4275726e206661696c65640000000000000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f496e636f72726563742066656520616d6f756e740000000000000000000000006044820152fd5b6024887f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b9092610e1a610e10858885614f86565b6020810190615087565b80915060051b90808204602014901517156109e15760400190816040116109e157600191610e47916143c5565b930190610adc565b6024897f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601660248201527f6e6f206368616c6c656e6765207363686564756c6564000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600f60248201527f7072656d61747572652070726f6f6600000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f656d7074792070726f6f660000000000000000000000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20707260448201527f6f766520706f7373657373696f6e0000000000000000000000000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611062611055614144565b61105d6157a5565b614fc6565b80f35b614167565b503461028b5761107936614266565b919061108c61108783614f44565b6142ed565b818452600c60205273ffffffffffffffffffffffffffffffffffffffff918260408620541633036111cd578493818552600b60205260408520928354858116947fffffffffffffffffffffffff00000000000000000000000000000000000000009182339116179055600c60205260408720908154169055604051943385857f686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c98a80a460086020526040872054169182611144578680f35b823b156111c9578561119d819593899793889484967f4059b6d700000000000000000000000000000000000000000000000000000000865260048601526024850152336044850152608060648501526084840191614424565b03925af18015610d08576111b5575b80808080808680f35b6111be9061419f565b61028b57805f6111ac565b8680fd5b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f6e6c79207468652070726f706f7365642073746f726167652070726f76696460448201527f65722063616e20636c61696d2073746f726167652070726f766964657220726f60648201527f6c650000000000000000000000000000000000000000000000000000000000006084820152fd5b503461028b5761128636613f7c565b61129861080082969593961115614352565b6112a461108785614f44565b8215611ad357838652600b60205273ffffffffffffffffffffffffffffffffffffffff6040872054163303611a4f578386526005602052604086205494846112eb8561472b565b885b868110611569575061132e7fd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed85991604051918291602083526020830190614111565b0390a2848752600860205273ffffffffffffffffffffffffffffffffffffffff6040882054169384611366575b602087604051908152f35b843b156115655794929091879492866040519788967f545f6ec5000000000000000000000000000000000000000000000000000000008852608488019060048901528a6024890152608060448901525260a486019060a48160051b88010194809289915b83831061144b57505050505061140f8387938795937ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc87809603016064860152614424565b03925af1801561144057611428575b808080808061135b565b611432839161419f565b61143c578161141e565b5080fd5b6040513d85823e3d90fd5b929597995092977fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5c9087929597030183528735907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc18336030182121561156157828201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18085850136030182121561155d576040835284840182013603018484018201351215611559578284010180350167ffffffffffffffff8135116115595780353603602082011361155957600192826020808761153e8297968360408199015260608601908481359101614424565b94010135910152990193019301899795938c999795926113ca565b8d80fd5b8e80fd5b8c80fd5b8780fd5b909150611577818787614f86565b8035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe181360301821215611a4b5760206115b3848a8a614f86565b013591601f83166119e657821561198157660400000000000083116118f657908b9392918a8552600560205260408520918254926115f0846147a5565b90556116036115fe846143b7565b615935565b8c8560051c9188915b8183106118b057915050875260046020526040872084885260205260408720558b86526002602052604086208387526020526040862091808201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18284013603018112156115655767ffffffffffffffff8183850101351161156557828201810180353603602090910113611565576116a684546144b2565b601f8111611871575b508790601f818486010135116001146117925760019891818486010135611760575b5082817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9394860101358a1b9401013560031b1c19161790555b898c52600360205260408c20908c526020528060051c60408c2055888b52600660205261174060408c209160051c82546143c5565b905561174c818a6143c5565b61175682856146a0565b52019086916112ed565b8484018201016020013591507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6116d1565b848952602089209092915b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0848487010135168a1061185357600199508483018401357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe081161061180f575b50508792010135811b01905561170b565b60207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60f886868901013560031b161c199185858801010101351690555f806117fe565b6020838601850182018101358355998a01996001909201910161179d565b8489526020808a206118a092868601850135601f810160051c830193116118a6575b601f0160051c0190615815565b5f6116af565b9091508190611893565b9260408395969798999a6118e693956118cd600180971b8c614600565b90825260046020528282209082526020522054906143c5565b92018f979695949392918e61160c565b60a484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152602160448201527f50696563652073697a65206d757374206265206c657373207468616e20325e3560648201527f30000000000000000000000000000000000000000000000000000000000000006084820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601b60448201527f53697a65206d7573742062652067726561746572207468616e203000000000006064820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601d60448201527f53697a65206d7573742062652061206d756c7469706c65206f662033320000006064820152fd5b8a80fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20616460448201527f64207069656365730000000000000000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601b60248201527f4d75737420616464206174206c65617374206f6e6520706965636500000000006044820152fd5b503461028b57611b4036613fea565b818392935260209260058452611b5960408420546150db565b610100908103908111611d0357818452600985526040842054907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91828101908111611cd65790611baa91846152ac565b93858501519083815260038752604081208651825287526040812054928301928311611ca9575003611bff5781611be091614462565b9182611bf3575b50506040519015158152f35b51101590505f80611be7565b60a484604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152604160248201527f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e207760448201527f697468207468652076657279206c617374206c656166206f662061207069656360648201527f65000000000000000000000000000000000000000000000000000000000000006084820152fd5b807f4e487b7100000000000000000000000000000000000000000000000000000000602492526011600452fd5b6024867f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b6024847f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020611d6d600435614f44565b6040519015158152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051818152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611de3614144565b9067ffffffffffffffff9060243582811161143c57611e06903690600401613f4e565b9390611e16610800861115614352565b67016345785d8a00009182341061203a575f8080808673ff000000000000000000000000000000000000635af1611e4b614b26565b5015610d17576001547fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000086821696611e8288614ec6565b169116176001558484526020956006875284604081205560078752846040812055600b875260408520917fffffffffffffffffffffffff00000000000000000000000000000000000000009233848254161790556008885273ffffffffffffffffffffffffffffffffffffffff6040872091168093825416179055600d875284604081205581611fb6575b50505033837f11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b962818480a33411611f46575b50604051908152f35b7ffffffffffffffffffffffffffffffffffffffffffffffffffe9cba87a2760000340190348211611f8957808080611f8394335af1610c79614b26565b5f611f3d565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b813b15610d135791849161200e93836040518096819582947f101c1eab0000000000000000000000000000000000000000000000000000000084528c6004850152336024850152606060448501526064840191614424565b03925af1801561144057908391612026575b80611f0d565b61202f9061419f565b61143c57815f612020565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f737962696c20666565206e6f74206d65740000000000000000000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876040516120d6816141cf565b600581527f352e302e30000000000000000000000000000000000000000000000000000000602082015260405191829160208352602083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57604060209160043561215861108782614f44565b8152600683522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040516107d08152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576121da6157a5565b7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c1680156122e6575b6103fb577fffffffffffffffffffffffffffffffffffffffffffffff0000000000000000006002917f2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c36122ab61225f6142b4565b73ffffffffffffffffffffffffffffffffffffffff7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541660405192839260408452604084019061401e565b9060208301520390a1161790557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160028152a180f35b50600267ffffffffffffffff8216101561220b565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602073ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993005416604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405166040000000000008152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356123f061108782614f44565b8152600983522054604051908152f35b503461028b5761240f36614266565b9061241e610800831115614352565b67ffffffffffffffff600154168310156125fb57828452602090600b825273ffffffffffffffffffffffffffffffffffffffff92836040872054163303612577579085918583526006845260408320948386549655600b8552604084207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905560078552836040812055600d85528360408120556008855260408420541691826124f6575b505050507f14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb791604051908152a280f35b823b15610d045761254d928492836040518096819582947f2abd465c0000000000000000000000000000000000000000000000000000000084528d60048501528c6024850152606060448501526064840191614424565b03925af18015610d0857612563575b80806124c6565b61256c9061419f565b610d0457835f61255c565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20646560448201527f6c657465206461746120736574730000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601960248201527f6461746120736574206964206f7574206f6620626f756e6473000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576126906157a5565b5f73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993008054907fffffffffffffffffffffffff000000000000000000000000000000000000000082169055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e08280a380f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043561275661108782614f44565b8152600a6020526040812080549061276d8261472b565b925b82811061278c576040516020808252819061028790820187614111565b806127996001928461440f565b90549060031b1c6127aa82876146a0565b520161276f565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356127f561108782614f44565b8152600783522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173a2aa501b19aff244d90cc15a4cf739d2725b57298152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051620151808152f35b503461028b5760209060207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576004359080926128d361108784614f44565b8282526005602052604082205492825b8481106128f557602086604051908152f35b818452600383526040842081855283526040842054612917575b6001016128e3565b946129236001916147a5565b95905061290f565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff7f00000000000000000000000000000000000000000000000000000000000000001630036129c15760206040517f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8152f35b60046040517fe07c8dba000000000000000000000000000000000000000000000000000000008152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576040612a24614d26565b67ffffffffffffffff83519216825260030b6020820152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57612a70614144565b602491823567ffffffffffffffff811161143c573660238201121561143c57806004013592612a9e8461422c565b612aab60405191826141eb565b8481526020948582019336888383010111612cf7578186928989930187378301015273ffffffffffffffffffffffffffffffffffffffff807f000000000000000000000000000000000000000000000000000000000000000016803014908115612d02575b506129c157612b1d6157a5565b821694604051907f52d1902d00000000000000000000000000000000000000000000000000000000825280826004818a5afa9182918793612cce575b5050612b8f578686604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b8590877f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc91828103612ca05750843b15612c71575080547fffffffffffffffffffffffff000000000000000000000000000000000000000016821790556040518592917fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b8480a2815115612c3b5750612c379382915190845af4612c31614b26565b91615895565b5080f35b935050505034612c49575080f35b807fb398979f0000000000000000000000000000000000000000000000000000000060049252fd5b82604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b604051907faa1d49a40000000000000000000000000000000000000000000000000000000082526004820152fd5b9080929350813d8311612cfb575b612ce681836141eb565b81010312612cf75751905f80612b59565b8580fd5b503d612cdc565b9050817f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541614155f612b10565b503461028b57612d3f36613fea565b81835260096020526040832054917f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83168303611d035790602093612d9d9392610b796040612d8c614d26565b9490938152600d8952205443614600565b604051908152f35b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760443567ffffffffffffffff811161143c57612df5903690600401613f4e565b90612e04610800831115614352565b6004358352600b60205273ffffffffffffffffffffffffffffffffffffffff60408420541633036134155760043583526006602052604083205415613391576004358352600d60205260408320805415613388575b50600a602052604083208054806130b5575b505060043583526006602052604083205460096020526040842055612e918354436143c5565b6024351061300b5782916004358352600760205260243560408420556006602052604083205415612fc9575b600860205273ffffffffffffffffffffffffffffffffffffffff6040842054169081612f2d575b8360043581526006602052604081205460405190602435825260208201527fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e604060043592a280f35b600760205260408420549160066020526040852054813b15612cf757858094612f9f604051978896879586947faa27ebcc000000000000000000000000000000000000000000000000000000008652600435600487015260248601526044850152608060648501526084840191614424565b03925af18015610d0857612fb5575b8080612ee4565b612fbe9061419f565b61028b57805f612fae565b6004357f02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c28428480a2600d6020528260408120556007602052826040812055612ebd565b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f6368616c6c656e67652065706f6368206d757374206265206174206c6561737460448201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560648201527f20667574757265000000000000000000000000000000000000000000000000006084820152fd5b6130be8161472b565b91855b8281106132e75750505090916130db611087600435614f44565b83805b8351821015613282576130f182856146a0565b5194600435875260036020526040872086885260205260408720549386916004358952600560205261312660408a20546150db565b9661010088810311613255579261313f6115fe8a6143b7565b88610100038111158061323e575b1561319b57906001613186926004358d5260046020528c8360408220915260205260408d2061317d8b8254614600565b90551b906143c5565b926131936115fe856143b7565b93909361313f565b505095509295936001926131f592956004358a52600360205260408a20818b526020525f60408b20556004358a52600260205260408a20908a5260205288604081206131e781546144b2565b80613200575b5050506143c5565b9401909192936130de565b601f808211881461321a5750505f9150555b885f806131ed565b916132375f929382865260208620940160051c8401898501615815565b5555613212565b506004358b52600560205260408b2054821061314d565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b905092919092600435855260066020526132a160408620918254614600565b90557f6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b160405160208152806132dd600435946020830190614111565b0390a25f80612e6b565b81547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808201918083116132555761331f838661440f565b919054600392831b1c613332868a6146a0565b521561335b57908291613348600195948761440f565b81939154921b1b191690558355016130c1565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526031600452fd5b4390555f612e59565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602c60248201527f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c65617660448201527f65732061726520616464656400000000000000000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603960248201527f6f6e6c79207468652073746f726167652070726f76696465722063616e206d6f60448201527f766520746f206e6578742070726f76696e6720706572696f64000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020612d9d600435614b55565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602067ffffffffffffffff60015416604051908152f35b503461028b5760407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043560243573ffffffffffffffffffffffffffffffffffffffff8082168092036136ae5761357d61108784614f44565b828452600b6020526040842054163381036136045781036135cd57508152600c602052604081207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905580f35b908252600c60205260408220907fffffffffffffffffffffffff000000000000000000000000000000000000000082541617905580f35b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604460248201527f4f6e6c79207468652063757272656e742073746f726167652070726f7669646560448201527f722063616e2070726f706f73652061206e65772073746f726167652070726f7660648201527f69646572000000000000000000000000000000000000000000000000000000006084820152fd5b5f80fd5b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576136f36044356024356004356147d2565b9160409491945193608085019260808652815180945260a086019360a08160051b88010194602080940192905b82821061375957888703858a015288808961374d8d61373f8c8c614111565b908482036040860152614111565b90151560608301520390f35b90919295848061379b837fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff608d60019603018652828b515191818152019061401e565b980192019201909291613720565b503461028b576137b83661407a565b909180845260056020526137cf60408520546150db565b91610100928303928311613832576137e681614625565b945b8181106137fd576040518061028788826140c8565b806138168561380f600194868a6143d2565b35866152ac565b61382082896146a0565b5261382b81886146a0565b50016137e8565b6024857f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff60406020926004356138b861108782614f44565b815260088452205416604051908152f35b503461028b576102876138e46138de36613fea565b90614503565b6040519182916020835251602080840152604083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760409060043561394161108782614f44565b8152600b60205273ffffffffffffffffffffffffffffffffffffffff8281818420541692600c60205220541682519182526020820152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356139bd61108782614f44565b8152600583522054604051908152f35b503461028b576020611d6d6139e136613fea565b90614462565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040517f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e8152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173fe000000000000000000000000000000000000068152f35b503461028b576040602091613aa136613fea565b90613aae61108782614f44565b82526003845282822090825283522054604051908152f35b50346136ae57613ad536613f7c565b90613ae7610800839594951115614352565b613af361108786614f44565b845f526020600b815273ffffffffffffffffffffffffffffffffffffffff908160405f2054163303613dc157865f52600a8082526107d0613b3860405f2054896143c5565b11613d3d575f5b878110613c26575050600890875f525260405f2054169182613b5f578680f35b823b156136ae57604051957fe7954aa70000000000000000000000000000000000000000000000000000000087526004870152606060248701528460648701527f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff85116136ae5785613bf38195935f9793608484968a9660051b809183880137850160808682030160448701520191614424565b03925af18015613c1b57613c0a5780808080808680f35b613c14915061419f565b5f806111ac565b6040513d5f823e3d90fd5b613c318189896143d2565b35895f526005845260405f20541115613cb957885f5281835260405f2090613c5a818a8a6143d2565b3582549268010000000000000000841015610a995783613c80916001809601815561440f565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff829392549160031b92831b921b191617905501613b3f565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602c60248201527f43616e206f6e6c79207363686564756c652072656d6f76616c206f662065786960448201527f7374696e672070696563657300000000000000000000000000000000000000006064820152fd5b608482604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603a60248201527f546f6f206d616e792072656d6f76616c73207761697420666f72206e6578742060448201527f70726f76696e6720706572696f6420746f207363686564756c650000000000006064820152fd5b608490604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20736360448201527f686564756c652072656d6f76616c206f662070696563657300000000000000006064820152fd5b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57602060405173ff000000000000000000000000000000000000638152f35b346136ae5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57600435613ecf61108782614f44565b5f52600d602052602060405f2054604051908152f35b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae578061080060209252f35b9181601f840112156136ae5782359167ffffffffffffffff83116136ae576020808501948460051b0101116136ae57565b9181601f840112156136ae5782359167ffffffffffffffff83116136ae57602083818601950101116136ae57565b9060607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae576004359167ffffffffffffffff916024358381116136ae5782613fcc91600401613f1d565b939093926044359182116136ae57613fe691600401613f4e565b9091565b7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc60409101126136ae576004359060243590565b91908251928382525f5b8481106140665750507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f845f6020809697860101520116010190565b602081830181015184830182015201614028565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f1d565b60208082019080835283518092528060408094019401925f905b8382106140f157505050505090565b8451805187528301518684015294850194938201936001909101906140e2565b9081518082526020808093019301915f5b828110614130575050505090565b835185529381019392810192600101614122565b6004359073ffffffffffffffffffffffffffffffffffffffff821682036136ae57565b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae5760206040515f8152f35b67ffffffffffffffff8111610a9957604052565b6020810190811067ffffffffffffffff821117610a9957604052565b6040810190811067ffffffffffffffff821117610a9957604052565b90601f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0910116810190811067ffffffffffffffff821117610a9957604052565b67ffffffffffffffff8111610a9957601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01660200190565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f4e565b604051906142c1826141cf565b600582527f322e302e300000000000000000000000000000000000000000000000000000006020830152565b156142f457565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f4461746120736574206e6f74206c6976650000000000000000000000000000006044820152fd5b1561435957565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f4578747261206461746120746f6f206c617267650000000000000000000000006044820152fd5b9060018201809211611f8957565b91908201809211611f8957565b91908110156143e25760051b0190565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b80548210156143e2575f5260205f2001905f90565b601f82602094937fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe093818652868601375f8582860101520116010190565b9061446c82614f44565b918261449b575b8261447d57505090565b9091505f52600360205260405f20905f5260205260405f2054151590565b8092505f52600560205260405f2054811091614473565b90600182811c921680156144f9575b60208310146144cc57565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b91607f16916144c1565b6060604051614511816141b3565b5261451e61108782614f44565b5f526020906002825260405f20905f52815260405f2060405191614541836141b3565b60405180925f908054614553816144b2565b808552916001918083169081156145bf5750600114614581575b50505061457c925003826141eb565b815290565b5f90815285812095935091905b8183106145a757505061457c93508201015f808061456d565b8554878401850152948501948694509183019161458e565b91505061457c9593507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f808061456d565b91908203918211611f8957565b67ffffffffffffffff8111610a995760051b60200190565b9061462f8261460d565b60409061463f60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe061466d829561460d565b01915f5b83811061467e5750505050565b602090825161468c816141cf565b5f8152825f81830152828601015201614671565b80518210156143e25760209160051b010190565b906146be8261460d565b6040906146ce60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe06146fc829561460d565b01915f5b83811061470d5750505050565b602090825161471b816141b3565b6060815282828601015201614700565b906147358261460d565b61474260405191826141eb565b8281527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0614770829461460d565b0190602036910137565b906701518000000000009180830292830403611f8957565b81810292918115918404141715611f8957565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8114611f895760010190565b915f6147e061108785614f44565b8115614ac857835f52600560205260405f2054926147fd836146b4565b926148078161472b565b946148118261472b565b965f905f945f5b848110614905575b505050505081155f1461486d5750505050505060405161483f816141b3565b5f81526040519161484f836141b3565b5f83526040519161485f836141b3565b5f83525f3681379291905f90565b81969293949596105f146148ff57614884816146b4565b9561488e8261472b565b956148988361472b565b955f5b8481106148a9575050505050565b806148b6600192846146a0565b516148c1828d6146a0565b526148cc818c6146a0565b506148d781856146a0565b516148e2828c6146a0565b526148ed81866146a0565b516148f8828b6146a0565b520161489b565b50919391565b825f52600360205260405f20815f5260205260405f2054614929575b600101614818565b92958187101580614abf575b15614a8e57825f52600260205260405f20845f5260205260405f206040519061495d826141b3565b60405190815f82549261496f846144b2565b8084529360018116908115614a4e5750600114614a0c575b50614994925003826141eb565b81526149a0828b6146a0565b526149ab818a6146a0565b50836149b7828c6146a0565b52825f52600360205260405f20845f5260205260405f2054908160051b9180830460201490151715611f89576149fe816001936149f88f94614a04956146a0565b526147a5565b976147a5565b939050614921565b9150505f528160205f20915f925b818410614a325750506020614994928201015f614987565b6020919250806001915483868801015201920191908391614a1a565b602093506149949592507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f614987565b95614a9986836143c5565b811015614aab57614a046001916147a5565b505050505091506001915f80808080614820565b50858110614935565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601c60248201527f4c696d6974206d7573742062652067726561746572207468616e2030000000006044820152fd5b3d15614b50573d90614b378261422c565b91614b4560405193846141eb565b82523d5f602084013e565b606090565b5f80916040516020810191825260208152614b6f816141cf565b519073fe000000000000000000000000000000000000065afa614b90614b26565b9015614baa576020818051810103126136ae576020015190565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c6560448201527f64000000000000000000000000000000000000000000000000000000000000006064820152fd5b908160809103126136ae576040519067ffffffffffffffff6080830181811184821017610a995760405281518060070b81036136ae578352602082015190811681036136ae5760208301526040810151908160030b82036136ae5760609160408401520151606082015290565b15614ca257565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f6661696c656420746f2076616c69646174653a207072696365206d757374206260448201527f652067726561746572207468616e2030000000000000000000000000000000006064820152fd5b6040908151917fa4ae35e00000000000000000000000000000000000000000000000000000000083527f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e9283600482015262015180602482015260809373a2aa501b19aff244d90cc15a4cf739d2725b5729918581604481865afa5f9181614ea7575b50614e92575084907f32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120614df0614ddd614b26565b865191829160208352602083019061401e565b0390a160248451809481937f96834ad300000000000000000000000000000000000000000000000000000000835260048301525afa938415614e88575f94614e59575b5050614e445f845160070b13614c9b565b825167ffffffffffffffff1692015160030b90565b614e79929450803d10614e81575b614e7181836141eb565b810190614c2e565b915f80614e33565b503d614e67565b82513d5f823e3d90fd5b809550614e4492505f91505160070b13614c9b565b614ebf919250873d8911614e8157614e7181836141eb565b905f614da9565b67ffffffffffffffff809116908114611f895760010190565b15614ee657565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601060248201527f5472616e73666572206661696c65642e000000000000000000000000000000006044820152fd5b67ffffffffffffffff6001541681109081614f5d575090565b90505f52600b60205273ffffffffffffffffffffffffffffffffffffffff60405f205416151590565b91908110156143e25760051b810135907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc1813603018212156136ae570190565b73ffffffffffffffffffffffffffffffffffffffff809116908115615057577f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300805490837fffffffffffffffffffffffff00000000000000000000000000000000000000008316179055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e05f80a3565b60246040517f1e4fbdf70000000000000000000000000000000000000000000000000000000081525f6004820152fd5b9035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe1813603018212156136ae570180359067ffffffffffffffff82116136ae57602001918160051b360383136136ae57565b610100908060801c806152a0575b508060401c8061526d575b508060201c8061523a575b508060101c80615207575b508060081c806151d4575b508060041c806151a1575b508060021c8061516e575b508060011c6151405761513d91614600565b90565b507ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe810191508111611f8957905f61512b565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc810191508111611f8957905f615120565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8810191508111611f8957905f615115565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0810191508111611f8957905f61510a565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0810191508111611f8957905f6150ff565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc0810191508111611f8957905f6150f4565b9150506080905f6150e9565b916040918251916152bc836141cf565b5f83525f602080940152845f5260068352835f2054821015615452576001947fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86831b818101908111611f895796905f9391825b61536e575050505f5260048352835f20855f528352615332845f2054826143c5565b82811115615358575061534491614600565b915192615350846141cf565b835282015290565b905060018501809511611f895761534491614600565b909197835f5260058752875f205481101561543b57835f52600494858852885f20825f528852866153a28a5f2054836143c5565b1161541b576153c390855f52868952895f20835f528952895f2054906143c5565b94838a01908a82116153ef575090826153dd921b906143c5565b975b8015611f89578201919082615310565b6011907f4e487b71000000000000000000000000000000000000000000000000000000005f525260245ffd5b94838a01908a82116153ef57509082615435921b90614600565b976153df565b82890190898211611f895782615435921b90614600565b6064838551907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152601860248201527f4c65616620696e646578206f7574206f6620626f756e647300000000000000006044820152fd5b63ffffffff16604d8111611f8957600a0a90565b93929190841580159061579d575b156157195767ffffffffffffffff1690811561569557821561561157600381900b5f81126155a3575061551561550961551b9361477a565b9163ffffffff166154af565b90614792565b8015610a6c5761553d9261553891671bc16d674ec8000004614792565b614792565b9060058202821590838104600514821715611f895760649004908360021b848104600414821715611f89578284106155785750505050505f90565b60649004831061558d575061513d9250614600565b9150508180046001141715611f89576064900490565b90507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008114611f89576155dd905f0363ffffffff166154af565b90671bc16d674ec800009180830292830403611f89576155fc9061477a565b928315610a6c5761553d936155389204614792565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603360248201527f6661696c656420746f2076616c69646174653a207261772073697a65206d757360448201527f742062652067726561746572207468616e2030000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603860248201527f6661696c656420746f2076616c69646174653a204174746f46494c207072696360448201527f65206d7573742062652067726561746572207468616e203000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603c60248201527f6661696c656420746f2076616c69646174653a20657374696d6174656420676160448201527f7320666565206d7573742062652067726561746572207468616e2030000000006064820152fd5b5048156154d1565b73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300541633036157e557565b60246040517f118cdaa7000000000000000000000000000000000000000000000000000000008152336004820152fd5b818110615820575050565b5f8155600101615815565b9081518110156143e2570160200190565b60ff7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a005460401c161561586b57565b60046040517fd7e6bcf8000000000000000000000000000000000000000000000000000000008152fd5b906158d457508051156158aa57805190602001fd5b60046040517fd6bda275000000000000000000000000000000000000000000000000000000008152fd5b8151158061592c575b6158e5575090565b60249073ffffffffffffffffffffffffffffffffffffffff604051917f9996b315000000000000000000000000000000000000000000000000000000008352166004820152fd5b50803b156158dd565b7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8111615c4057610100907f80000000000000000000000000000000000000000000000000000000000000008114611f8957805f031680615c37575b6fffffffffffffffffffffffffffffffff8116615c06575b77ffffffffffffffff0000000000000000ffffffffffffffff8116615bd5575b7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff8116615ba4575b7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff8116615b73575b7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff8116615b42575b7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f8116615b11575b7f33333333333333333333333333333333333333333333333333333333333333338116615ae0575b7f555555555555555555555555555555555555555555555555555555555555555516615ab35790565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8101908111611f895790565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790615a8a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8101908111611f895790615a62565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff88101908111611f895790615a3a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff08101908111611f895790615a13565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101908111611f8957906159ed565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc08101908111611f8957906159c9565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808101908111611f8957906159a9565b60ff9150615991565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f496e7075742065786365656473206d6178696d756d20696e743235362076616c60448201527f75650000000000000000000000000000000000000000000000000000000000006064820152fd5b5f5260205260205f60408160025afa156136ae577fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f5f51169056fea2646970667358221220a2d6c54c44a68038f7d59aa211cb3b39a5f231a0e7b9f3f652141521dc58b6fd64736f6c63430008170033","sourceMap":"2048:37005:43:-:0;;;;;;;1171:4:25;1163:13;;8837:64:24;2048:37005:43;;;;;;;;;7896:76:24;;-1:-1:-1;;;;;;;;;;;;2048:37005:43;;;7985:34:24;7981:146;;-1:-1:-1;2048:37005:43;;;;;;;;1163:13:25;2048:37005:43;;;;;;;;;;;7981:146:24;-1:-1:-1;;;;;;2048:37005:43;;;;;;;;;;;;;8087:29:24;;2048:37005:43;;8087:29:24;7981:146;;;;;7896:76;-1:-1:-1;;;7938:23:24;;;;;2048:37005:43;;;","linkReferences":{}},"deployedBytecode":{"object":"0x610120806040526004361015610013575f80fd5b5f905f3560e01c908163029b464614613ee55750806304595c1a14613e915780630a6a63f114613e455780630c29202414613ac65780630cd7b88014613a8d57806315b1757014613a4057806319c75950146139e75780631a271225146139cd5780631c5ae80f1461397957806321b7cd1c146138ff57806325bbbedf146138c95780632b3129bb1461385f578063349c9179146137a957806339f51544146136b2578063431860801461351c578063442cded3146134d6578063453f4f621461349957806345c0b92d14612da5578063462dd449146110655780634903704a14612d305780634f1ef28614612a3d5780634fa27920146129eb57806352d1902d1461292b5780635353bdfd1461288e57806361a52a361461285257806367e406d5146128055780636ba4608f146127b15780636fa4469214612717578063715018a6146126595780637a1e29901461240057806389208ba9146123ac5780638a405abc1461236c5780638da5cb5b146122fb5780638fd3ab80146121a35780639f8cb3bd14612168578063a531998c14612114578063ad3cb1cc14612098578063bbae41cb14611db0578063c0e1594914611d77578063ca759f2714611d30578063dc63526614611b31578063ddea76cc14611277578063df0f32481461106a578063f178b1be14611065578063f2fde38b1461101a578063f58f952b1461047b578063f83758fe14610440578063fe4b84df1461028e5763ffa1ad741461023a575f80fd5b3461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876102736142b4565b60405191829160208352602083019061401e565b0390f35b80fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b577ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c16159067ffffffffffffffff811680159081610438575b600114908161042e575b159081610425575b506103fb578160017fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000083161784556103c6575b5061034c61583c565b61035461583c565b61035d33614fc6565b61036561583c565b6004358355610372575080f35b7fffffffffffffffffffffffffffffffffffffffffffffff00ffffffffffffffff81541690557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160018152a180f35b7fffffffffffffffffffffffffffffffffffffffffffffff00000000000000000016680100000000000000011782555f610343565b60046040517ff92ee8a9000000000000000000000000000000000000000000000000000000008152fd5b9050155f610310565b303b159150610308565b8391506102fe565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760209054604051908152f35b506104853661407a565b9291905a93828452600b60205273ffffffffffffffffffffffffffffffffffffffff92836040862054163303610f96578115610f385780855260076020526040852054804310610eda5715610e7c576104dd82614625565b9381865260076020526104f36040872054614b55565b8287526009602052604087205490600560205261051360408920546150db565b9261010084810311610e4f579193888888888d9484985b8267ffffffffffffffff8b161015610ac6576040518860208201528260408201527fffffffffffffffff0000000000000000000000000000000000000000000000008b60c01b1660608201526048815280608081011067ffffffffffffffff608083011117610a995760808101604052602081519101208b15610a6c576105b9908c83610100039106846152ac565b6105cd67ffffffffffffffff8c16876146a0565b526105e267ffffffffffffffff8b16866146a0565b506106026105fa67ffffffffffffffff8c16876146a0565b515183614503565b602081515110610a0e5760405190610619826141cf565b6020825260203681840137875b602081106109455750508660c0526020815191015160c0526020811061090f575b5081865260036020526040862061066867ffffffffffffffff8c16876146a0565b5151875260205260408620547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff81019081116108e2576106a7906150db565b610100039a6101008c116108e25760018c018c116108e2578a9b8b602061070567ffffffffffffffff6106fc816106ee6106e58d8d848a1691614f86565b86810190615087565b906101005295168a8c614f86565b359f168a6146a0565b5101516080526107148161460d565b60e05260405160a05261072b60e05160a0516141eb565b60a051508060a05152602060a05101368260051b6101005101116108de5761010051905b8260051b610100510182106108ce5750505060a051510361084a57608051969a96988b975b60a051518d10156107bd5760019061078e8e60a0516146a0565b51908c83166107ae57906107a191615cc4565b9a5b811c9c019b99610774565b6107b791615cc4565b9a6107a3565b9195995093979b91959992969a5060c051036107ec576107dc90614ec6565b989490979399959196929961052a565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f70726f6f6620646964206e6f74207665726966790000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f70726f6f66206c656e67746820646f6573206e6f74206d61746368207472656560448201527f20686569676874000000000000000000000000000000000000000000000000006064820152fd5b813581526020918201910161074f565b8980fd5b6024877f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9060200360031b1b60c0511660c0528b610647565b81518051807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101116109e1576109cc83926109c66001957fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe07fff0000000000000000000000000000000000000000000000000000000000000095016143c5565b9061582b565b51168a1a6109da828661582b565b5301610626565b60248b7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601560248201527f436964206461746120697320746f6f2073686f727400000000000000000000006044820152fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b5093959091610ad788965a90614600565b879188905b858210610e0057505060208201809211610dd35761051491828102928184041490151715610dd357610b1891610b11916143c5565b4890614792565b84875260096020526040872054907f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82168203610dd357610b83610b799493926040928a610b64614d26565b988196838d8c9552600d602052205443614600565b9360051b926154c3565b803410610d75575f8080808473ff000000000000000000000000000000000000635af1610bae614b26565b5015610d1757867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b6060610c03948c9867ffffffffffffffff6040519287845216602083015260030b6040820152a234614600565b9585845260086020526040842054169081610c84575b5050507f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e509183610c5b9252600d602052436040872055604051918291826140c8565b0390a28181610c675780f35b808080610c7f94335af1610c79614b26565b50614edf565b818180f35b6006602052604084205491803b15610d135784928360849260405196879586947f356de02b0000000000000000000000000000000000000000000000000000000086528c60048701526024860152604485015260648401525af18015610d0857610cf0575b8080610c19565b610cf99061419f565b610d04578385610ce9565b8380fd5b6040513d84823e3d90fd5b8480fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f4275726e206661696c65640000000000000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f496e636f72726563742066656520616d6f756e740000000000000000000000006044820152fd5b6024887f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b9092610e1a610e10858885614f86565b6020810190615087565b80915060051b90808204602014901517156109e15760400190816040116109e157600191610e47916143c5565b930190610adc565b6024897f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601660248201527f6e6f206368616c6c656e6765207363686564756c6564000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600f60248201527f7072656d61747572652070726f6f6600000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f656d7074792070726f6f660000000000000000000000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20707260448201527f6f766520706f7373657373696f6e0000000000000000000000000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611062611055614144565b61105d6157a5565b614fc6565b80f35b614167565b503461028b5761107936614266565b919061108c61108783614f44565b6142ed565b818452600c60205273ffffffffffffffffffffffffffffffffffffffff918260408620541633036111cd578493818552600b60205260408520928354858116947fffffffffffffffffffffffff00000000000000000000000000000000000000009182339116179055600c60205260408720908154169055604051943385857f686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c98a80a460086020526040872054169182611144578680f35b823b156111c9578561119d819593899793889484967f4059b6d700000000000000000000000000000000000000000000000000000000865260048601526024850152336044850152608060648501526084840191614424565b03925af18015610d08576111b5575b80808080808680f35b6111be9061419f565b61028b57805f6111ac565b8680fd5b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f6e6c79207468652070726f706f7365642073746f726167652070726f76696460448201527f65722063616e20636c61696d2073746f726167652070726f766964657220726f60648201527f6c650000000000000000000000000000000000000000000000000000000000006084820152fd5b503461028b5761128636613f7c565b61129861080082969593961115614352565b6112a461108785614f44565b8215611ad357838652600b60205273ffffffffffffffffffffffffffffffffffffffff6040872054163303611a4f578386526005602052604086205494846112eb8561472b565b885b868110611569575061132e7fd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed85991604051918291602083526020830190614111565b0390a2848752600860205273ffffffffffffffffffffffffffffffffffffffff6040882054169384611366575b602087604051908152f35b843b156115655794929091879492866040519788967f545f6ec5000000000000000000000000000000000000000000000000000000008852608488019060048901528a6024890152608060448901525260a486019060a48160051b88010194809289915b83831061144b57505050505061140f8387938795937ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc87809603016064860152614424565b03925af1801561144057611428575b808080808061135b565b611432839161419f565b61143c578161141e565b5080fd5b6040513d85823e3d90fd5b929597995092977fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5c9087929597030183528735907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc18336030182121561156157828201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18085850136030182121561155d576040835284840182013603018484018201351215611559578284010180350167ffffffffffffffff8135116115595780353603602082011361155957600192826020808761153e8297968360408199015260608601908481359101614424565b94010135910152990193019301899795938c999795926113ca565b8d80fd5b8e80fd5b8c80fd5b8780fd5b909150611577818787614f86565b8035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe181360301821215611a4b5760206115b3848a8a614f86565b013591601f83166119e657821561198157660400000000000083116118f657908b9392918a8552600560205260408520918254926115f0846147a5565b90556116036115fe846143b7565b615935565b8c8560051c9188915b8183106118b057915050875260046020526040872084885260205260408720558b86526002602052604086208387526020526040862091808201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18284013603018112156115655767ffffffffffffffff8183850101351161156557828201810180353603602090910113611565576116a684546144b2565b601f8111611871575b508790601f818486010135116001146117925760019891818486010135611760575b5082817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9394860101358a1b9401013560031b1c19161790555b898c52600360205260408c20908c526020528060051c60408c2055888b52600660205261174060408c209160051c82546143c5565b905561174c818a6143c5565b61175682856146a0565b52019086916112ed565b8484018201016020013591507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6116d1565b848952602089209092915b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0848487010135168a1061185357600199508483018401357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe081161061180f575b50508792010135811b01905561170b565b60207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60f886868901013560031b161c199185858801010101351690555f806117fe565b6020838601850182018101358355998a01996001909201910161179d565b8489526020808a206118a092868601850135601f810160051c830193116118a6575b601f0160051c0190615815565b5f6116af565b9091508190611893565b9260408395969798999a6118e693956118cd600180971b8c614600565b90825260046020528282209082526020522054906143c5565b92018f979695949392918e61160c565b60a484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152602160448201527f50696563652073697a65206d757374206265206c657373207468616e20325e3560648201527f30000000000000000000000000000000000000000000000000000000000000006084820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601b60448201527f53697a65206d7573742062652067726561746572207468616e203000000000006064820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601d60448201527f53697a65206d7573742062652061206d756c7469706c65206f662033320000006064820152fd5b8a80fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20616460448201527f64207069656365730000000000000000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601b60248201527f4d75737420616464206174206c65617374206f6e6520706965636500000000006044820152fd5b503461028b57611b4036613fea565b818392935260209260058452611b5960408420546150db565b610100908103908111611d0357818452600985526040842054907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91828101908111611cd65790611baa91846152ac565b93858501519083815260038752604081208651825287526040812054928301928311611ca9575003611bff5781611be091614462565b9182611bf3575b50506040519015158152f35b51101590505f80611be7565b60a484604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152604160248201527f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e207760448201527f697468207468652076657279206c617374206c656166206f662061207069656360648201527f65000000000000000000000000000000000000000000000000000000000000006084820152fd5b807f4e487b7100000000000000000000000000000000000000000000000000000000602492526011600452fd5b6024867f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b6024847f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020611d6d600435614f44565b6040519015158152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051818152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611de3614144565b9067ffffffffffffffff9060243582811161143c57611e06903690600401613f4e565b9390611e16610800861115614352565b67016345785d8a00009182341061203a575f8080808673ff000000000000000000000000000000000000635af1611e4b614b26565b5015610d17576001547fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000086821696611e8288614ec6565b169116176001558484526020956006875284604081205560078752846040812055600b875260408520917fffffffffffffffffffffffff00000000000000000000000000000000000000009233848254161790556008885273ffffffffffffffffffffffffffffffffffffffff6040872091168093825416179055600d875284604081205581611fb6575b50505033837f11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b962818480a33411611f46575b50604051908152f35b7ffffffffffffffffffffffffffffffffffffffffffffffffffe9cba87a2760000340190348211611f8957808080611f8394335af1610c79614b26565b5f611f3d565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b813b15610d135791849161200e93836040518096819582947f101c1eab0000000000000000000000000000000000000000000000000000000084528c6004850152336024850152606060448501526064840191614424565b03925af1801561144057908391612026575b80611f0d565b61202f9061419f565b61143c57815f612020565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f737962696c20666565206e6f74206d65740000000000000000000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876040516120d6816141cf565b600581527f352e302e30000000000000000000000000000000000000000000000000000000602082015260405191829160208352602083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57604060209160043561215861108782614f44565b8152600683522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040516107d08152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576121da6157a5565b7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c1680156122e6575b6103fb577fffffffffffffffffffffffffffffffffffffffffffffff0000000000000000006002917f2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c36122ab61225f6142b4565b73ffffffffffffffffffffffffffffffffffffffff7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541660405192839260408452604084019061401e565b9060208301520390a1161790557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160028152a180f35b50600267ffffffffffffffff8216101561220b565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602073ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993005416604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405166040000000000008152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356123f061108782614f44565b8152600983522054604051908152f35b503461028b5761240f36614266565b9061241e610800831115614352565b67ffffffffffffffff600154168310156125fb57828452602090600b825273ffffffffffffffffffffffffffffffffffffffff92836040872054163303612577579085918583526006845260408320948386549655600b8552604084207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905560078552836040812055600d85528360408120556008855260408420541691826124f6575b505050507f14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb791604051908152a280f35b823b15610d045761254d928492836040518096819582947f2abd465c0000000000000000000000000000000000000000000000000000000084528d60048501528c6024850152606060448501526064840191614424565b03925af18015610d0857612563575b80806124c6565b61256c9061419f565b610d0457835f61255c565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20646560448201527f6c657465206461746120736574730000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601960248201527f6461746120736574206964206f7574206f6620626f756e6473000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576126906157a5565b5f73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993008054907fffffffffffffffffffffffff000000000000000000000000000000000000000082169055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e08280a380f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043561275661108782614f44565b8152600a6020526040812080549061276d8261472b565b925b82811061278c576040516020808252819061028790820187614111565b806127996001928461440f565b90549060031b1c6127aa82876146a0565b520161276f565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356127f561108782614f44565b8152600783522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173a2aa501b19aff244d90cc15a4cf739d2725b57298152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051620151808152f35b503461028b5760209060207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576004359080926128d361108784614f44565b8282526005602052604082205492825b8481106128f557602086604051908152f35b818452600383526040842081855283526040842054612917575b6001016128e3565b946129236001916147a5565b95905061290f565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff7f00000000000000000000000000000000000000000000000000000000000000001630036129c15760206040517f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8152f35b60046040517fe07c8dba000000000000000000000000000000000000000000000000000000008152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576040612a24614d26565b67ffffffffffffffff83519216825260030b6020820152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57612a70614144565b602491823567ffffffffffffffff811161143c573660238201121561143c57806004013592612a9e8461422c565b612aab60405191826141eb565b8481526020948582019336888383010111612cf7578186928989930187378301015273ffffffffffffffffffffffffffffffffffffffff807f000000000000000000000000000000000000000000000000000000000000000016803014908115612d02575b506129c157612b1d6157a5565b821694604051907f52d1902d00000000000000000000000000000000000000000000000000000000825280826004818a5afa9182918793612cce575b5050612b8f578686604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b8590877f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc91828103612ca05750843b15612c71575080547fffffffffffffffffffffffff000000000000000000000000000000000000000016821790556040518592917fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b8480a2815115612c3b5750612c379382915190845af4612c31614b26565b91615895565b5080f35b935050505034612c49575080f35b807fb398979f0000000000000000000000000000000000000000000000000000000060049252fd5b82604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b604051907faa1d49a40000000000000000000000000000000000000000000000000000000082526004820152fd5b9080929350813d8311612cfb575b612ce681836141eb565b81010312612cf75751905f80612b59565b8580fd5b503d612cdc565b9050817f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541614155f612b10565b503461028b57612d3f36613fea565b81835260096020526040832054917f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83168303611d035790602093612d9d9392610b796040612d8c614d26565b9490938152600d8952205443614600565b604051908152f35b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760443567ffffffffffffffff811161143c57612df5903690600401613f4e565b90612e04610800831115614352565b6004358352600b60205273ffffffffffffffffffffffffffffffffffffffff60408420541633036134155760043583526006602052604083205415613391576004358352600d60205260408320805415613388575b50600a602052604083208054806130b5575b505060043583526006602052604083205460096020526040842055612e918354436143c5565b6024351061300b5782916004358352600760205260243560408420556006602052604083205415612fc9575b600860205273ffffffffffffffffffffffffffffffffffffffff6040842054169081612f2d575b8360043581526006602052604081205460405190602435825260208201527fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e604060043592a280f35b600760205260408420549160066020526040852054813b15612cf757858094612f9f604051978896879586947faa27ebcc000000000000000000000000000000000000000000000000000000008652600435600487015260248601526044850152608060648501526084840191614424565b03925af18015610d0857612fb5575b8080612ee4565b612fbe9061419f565b61028b57805f612fae565b6004357f02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c28428480a2600d6020528260408120556007602052826040812055612ebd565b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f6368616c6c656e67652065706f6368206d757374206265206174206c6561737460448201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560648201527f20667574757265000000000000000000000000000000000000000000000000006084820152fd5b6130be8161472b565b91855b8281106132e75750505090916130db611087600435614f44565b83805b8351821015613282576130f182856146a0565b5194600435875260036020526040872086885260205260408720549386916004358952600560205261312660408a20546150db565b9661010088810311613255579261313f6115fe8a6143b7565b88610100038111158061323e575b1561319b57906001613186926004358d5260046020528c8360408220915260205260408d2061317d8b8254614600565b90551b906143c5565b926131936115fe856143b7565b93909361313f565b505095509295936001926131f592956004358a52600360205260408a20818b526020525f60408b20556004358a52600260205260408a20908a5260205288604081206131e781546144b2565b80613200575b5050506143c5565b9401909192936130de565b601f808211881461321a5750505f9150555b885f806131ed565b916132375f929382865260208620940160051c8401898501615815565b5555613212565b506004358b52600560205260408b2054821061314d565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b905092919092600435855260066020526132a160408620918254614600565b90557f6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b160405160208152806132dd600435946020830190614111565b0390a25f80612e6b565b81547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808201918083116132555761331f838661440f565b919054600392831b1c613332868a6146a0565b521561335b57908291613348600195948761440f565b81939154921b1b191690558355016130c1565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526031600452fd5b4390555f612e59565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602c60248201527f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c65617660448201527f65732061726520616464656400000000000000000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603960248201527f6f6e6c79207468652073746f726167652070726f76696465722063616e206d6f60448201527f766520746f206e6578742070726f76696e6720706572696f64000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020612d9d600435614b55565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602067ffffffffffffffff60015416604051908152f35b503461028b5760407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043560243573ffffffffffffffffffffffffffffffffffffffff8082168092036136ae5761357d61108784614f44565b828452600b6020526040842054163381036136045781036135cd57508152600c602052604081207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905580f35b908252600c60205260408220907fffffffffffffffffffffffff000000000000000000000000000000000000000082541617905580f35b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604460248201527f4f6e6c79207468652063757272656e742073746f726167652070726f7669646560448201527f722063616e2070726f706f73652061206e65772073746f726167652070726f7660648201527f69646572000000000000000000000000000000000000000000000000000000006084820152fd5b5f80fd5b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576136f36044356024356004356147d2565b9160409491945193608085019260808652815180945260a086019360a08160051b88010194602080940192905b82821061375957888703858a015288808961374d8d61373f8c8c614111565b908482036040860152614111565b90151560608301520390f35b90919295848061379b837fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff608d60019603018652828b515191818152019061401e565b980192019201909291613720565b503461028b576137b83661407a565b909180845260056020526137cf60408520546150db565b91610100928303928311613832576137e681614625565b945b8181106137fd576040518061028788826140c8565b806138168561380f600194868a6143d2565b35866152ac565b61382082896146a0565b5261382b81886146a0565b50016137e8565b6024857f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff60406020926004356138b861108782614f44565b815260088452205416604051908152f35b503461028b576102876138e46138de36613fea565b90614503565b6040519182916020835251602080840152604083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760409060043561394161108782614f44565b8152600b60205273ffffffffffffffffffffffffffffffffffffffff8281818420541692600c60205220541682519182526020820152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356139bd61108782614f44565b8152600583522054604051908152f35b503461028b576020611d6d6139e136613fea565b90614462565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040517f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e8152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173fe000000000000000000000000000000000000068152f35b503461028b576040602091613aa136613fea565b90613aae61108782614f44565b82526003845282822090825283522054604051908152f35b50346136ae57613ad536613f7c565b90613ae7610800839594951115614352565b613af361108786614f44565b845f526020600b815273ffffffffffffffffffffffffffffffffffffffff908160405f2054163303613dc157865f52600a8082526107d0613b3860405f2054896143c5565b11613d3d575f5b878110613c26575050600890875f525260405f2054169182613b5f578680f35b823b156136ae57604051957fe7954aa70000000000000000000000000000000000000000000000000000000087526004870152606060248701528460648701527f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff85116136ae5785613bf38195935f9793608484968a9660051b809183880137850160808682030160448701520191614424565b03925af18015613c1b57613c0a5780808080808680f35b613c14915061419f565b5f806111ac565b6040513d5f823e3d90fd5b613c318189896143d2565b35895f526005845260405f20541115613cb957885f5281835260405f2090613c5a818a8a6143d2565b3582549268010000000000000000841015610a995783613c80916001809601815561440f565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff829392549160031b92831b921b191617905501613b3f565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602c60248201527f43616e206f6e6c79207363686564756c652072656d6f76616c206f662065786960448201527f7374696e672070696563657300000000000000000000000000000000000000006064820152fd5b608482604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603a60248201527f546f6f206d616e792072656d6f76616c73207761697420666f72206e6578742060448201527f70726f76696e6720706572696f6420746f207363686564756c650000000000006064820152fd5b608490604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20736360448201527f686564756c652072656d6f76616c206f662070696563657300000000000000006064820152fd5b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57602060405173ff000000000000000000000000000000000000638152f35b346136ae5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57600435613ecf61108782614f44565b5f52600d602052602060405f2054604051908152f35b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae578061080060209252f35b9181601f840112156136ae5782359167ffffffffffffffff83116136ae576020808501948460051b0101116136ae57565b9181601f840112156136ae5782359167ffffffffffffffff83116136ae57602083818601950101116136ae57565b9060607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae576004359167ffffffffffffffff916024358381116136ae5782613fcc91600401613f1d565b939093926044359182116136ae57613fe691600401613f4e565b9091565b7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc60409101126136ae576004359060243590565b91908251928382525f5b8481106140665750507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f845f6020809697860101520116010190565b602081830181015184830182015201614028565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f1d565b60208082019080835283518092528060408094019401925f905b8382106140f157505050505090565b8451805187528301518684015294850194938201936001909101906140e2565b9081518082526020808093019301915f5b828110614130575050505090565b835185529381019392810192600101614122565b6004359073ffffffffffffffffffffffffffffffffffffffff821682036136ae57565b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae5760206040515f8152f35b67ffffffffffffffff8111610a9957604052565b6020810190811067ffffffffffffffff821117610a9957604052565b6040810190811067ffffffffffffffff821117610a9957604052565b90601f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0910116810190811067ffffffffffffffff821117610a9957604052565b67ffffffffffffffff8111610a9957601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01660200190565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f4e565b604051906142c1826141cf565b600582527f322e302e300000000000000000000000000000000000000000000000000000006020830152565b156142f457565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f4461746120736574206e6f74206c6976650000000000000000000000000000006044820152fd5b1561435957565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f4578747261206461746120746f6f206c617267650000000000000000000000006044820152fd5b9060018201809211611f8957565b91908201809211611f8957565b91908110156143e25760051b0190565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b80548210156143e2575f5260205f2001905f90565b601f82602094937fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe093818652868601375f8582860101520116010190565b9061446c82614f44565b918261449b575b8261447d57505090565b9091505f52600360205260405f20905f5260205260405f2054151590565b8092505f52600560205260405f2054811091614473565b90600182811c921680156144f9575b60208310146144cc57565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b91607f16916144c1565b6060604051614511816141b3565b5261451e61108782614f44565b5f526020906002825260405f20905f52815260405f2060405191614541836141b3565b60405180925f908054614553816144b2565b808552916001918083169081156145bf5750600114614581575b50505061457c925003826141eb565b815290565b5f90815285812095935091905b8183106145a757505061457c93508201015f808061456d565b8554878401850152948501948694509183019161458e565b91505061457c9593507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f808061456d565b91908203918211611f8957565b67ffffffffffffffff8111610a995760051b60200190565b9061462f8261460d565b60409061463f60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe061466d829561460d565b01915f5b83811061467e5750505050565b602090825161468c816141cf565b5f8152825f81830152828601015201614671565b80518210156143e25760209160051b010190565b906146be8261460d565b6040906146ce60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe06146fc829561460d565b01915f5b83811061470d5750505050565b602090825161471b816141b3565b6060815282828601015201614700565b906147358261460d565b61474260405191826141eb565b8281527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0614770829461460d565b0190602036910137565b906701518000000000009180830292830403611f8957565b81810292918115918404141715611f8957565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8114611f895760010190565b915f6147e061108785614f44565b8115614ac857835f52600560205260405f2054926147fd836146b4565b926148078161472b565b946148118261472b565b965f905f945f5b848110614905575b505050505081155f1461486d5750505050505060405161483f816141b3565b5f81526040519161484f836141b3565b5f83526040519161485f836141b3565b5f83525f3681379291905f90565b81969293949596105f146148ff57614884816146b4565b9561488e8261472b565b956148988361472b565b955f5b8481106148a9575050505050565b806148b6600192846146a0565b516148c1828d6146a0565b526148cc818c6146a0565b506148d781856146a0565b516148e2828c6146a0565b526148ed81866146a0565b516148f8828b6146a0565b520161489b565b50919391565b825f52600360205260405f20815f5260205260405f2054614929575b600101614818565b92958187101580614abf575b15614a8e57825f52600260205260405f20845f5260205260405f206040519061495d826141b3565b60405190815f82549261496f846144b2565b8084529360018116908115614a4e5750600114614a0c575b50614994925003826141eb565b81526149a0828b6146a0565b526149ab818a6146a0565b50836149b7828c6146a0565b52825f52600360205260405f20845f5260205260405f2054908160051b9180830460201490151715611f89576149fe816001936149f88f94614a04956146a0565b526147a5565b976147a5565b939050614921565b9150505f528160205f20915f925b818410614a325750506020614994928201015f614987565b6020919250806001915483868801015201920191908391614a1a565b602093506149949592507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f614987565b95614a9986836143c5565b811015614aab57614a046001916147a5565b505050505091506001915f80808080614820565b50858110614935565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601c60248201527f4c696d6974206d7573742062652067726561746572207468616e2030000000006044820152fd5b3d15614b50573d90614b378261422c565b91614b4560405193846141eb565b82523d5f602084013e565b606090565b5f80916040516020810191825260208152614b6f816141cf565b519073fe000000000000000000000000000000000000065afa614b90614b26565b9015614baa576020818051810103126136ae576020015190565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c6560448201527f64000000000000000000000000000000000000000000000000000000000000006064820152fd5b908160809103126136ae576040519067ffffffffffffffff6080830181811184821017610a995760405281518060070b81036136ae578352602082015190811681036136ae5760208301526040810151908160030b82036136ae5760609160408401520151606082015290565b15614ca257565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f6661696c656420746f2076616c69646174653a207072696365206d757374206260448201527f652067726561746572207468616e2030000000000000000000000000000000006064820152fd5b6040908151917fa4ae35e00000000000000000000000000000000000000000000000000000000083527f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e9283600482015262015180602482015260809373a2aa501b19aff244d90cc15a4cf739d2725b5729918581604481865afa5f9181614ea7575b50614e92575084907f32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120614df0614ddd614b26565b865191829160208352602083019061401e565b0390a160248451809481937f96834ad300000000000000000000000000000000000000000000000000000000835260048301525afa938415614e88575f94614e59575b5050614e445f845160070b13614c9b565b825167ffffffffffffffff1692015160030b90565b614e79929450803d10614e81575b614e7181836141eb565b810190614c2e565b915f80614e33565b503d614e67565b82513d5f823e3d90fd5b809550614e4492505f91505160070b13614c9b565b614ebf919250873d8911614e8157614e7181836141eb565b905f614da9565b67ffffffffffffffff809116908114611f895760010190565b15614ee657565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601060248201527f5472616e73666572206661696c65642e000000000000000000000000000000006044820152fd5b67ffffffffffffffff6001541681109081614f5d575090565b90505f52600b60205273ffffffffffffffffffffffffffffffffffffffff60405f205416151590565b91908110156143e25760051b810135907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc1813603018212156136ae570190565b73ffffffffffffffffffffffffffffffffffffffff809116908115615057577f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300805490837fffffffffffffffffffffffff00000000000000000000000000000000000000008316179055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e05f80a3565b60246040517f1e4fbdf70000000000000000000000000000000000000000000000000000000081525f6004820152fd5b9035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe1813603018212156136ae570180359067ffffffffffffffff82116136ae57602001918160051b360383136136ae57565b610100908060801c806152a0575b508060401c8061526d575b508060201c8061523a575b508060101c80615207575b508060081c806151d4575b508060041c806151a1575b508060021c8061516e575b508060011c6151405761513d91614600565b90565b507ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe810191508111611f8957905f61512b565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc810191508111611f8957905f615120565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8810191508111611f8957905f615115565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0810191508111611f8957905f61510a565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0810191508111611f8957905f6150ff565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc0810191508111611f8957905f6150f4565b9150506080905f6150e9565b916040918251916152bc836141cf565b5f83525f602080940152845f5260068352835f2054821015615452576001947fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86831b818101908111611f895796905f9391825b61536e575050505f5260048352835f20855f528352615332845f2054826143c5565b82811115615358575061534491614600565b915192615350846141cf565b835282015290565b905060018501809511611f895761534491614600565b909197835f5260058752875f205481101561543b57835f52600494858852885f20825f528852866153a28a5f2054836143c5565b1161541b576153c390855f52868952895f20835f528952895f2054906143c5565b94838a01908a82116153ef575090826153dd921b906143c5565b975b8015611f89578201919082615310565b6011907f4e487b71000000000000000000000000000000000000000000000000000000005f525260245ffd5b94838a01908a82116153ef57509082615435921b90614600565b976153df565b82890190898211611f895782615435921b90614600565b6064838551907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152601860248201527f4c65616620696e646578206f7574206f6620626f756e647300000000000000006044820152fd5b63ffffffff16604d8111611f8957600a0a90565b93929190841580159061579d575b156157195767ffffffffffffffff1690811561569557821561561157600381900b5f81126155a3575061551561550961551b9361477a565b9163ffffffff166154af565b90614792565b8015610a6c5761553d9261553891671bc16d674ec8000004614792565b614792565b9060058202821590838104600514821715611f895760649004908360021b848104600414821715611f89578284106155785750505050505f90565b60649004831061558d575061513d9250614600565b9150508180046001141715611f89576064900490565b90507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008114611f89576155dd905f0363ffffffff166154af565b90671bc16d674ec800009180830292830403611f89576155fc9061477a565b928315610a6c5761553d936155389204614792565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603360248201527f6661696c656420746f2076616c69646174653a207261772073697a65206d757360448201527f742062652067726561746572207468616e2030000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603860248201527f6661696c656420746f2076616c69646174653a204174746f46494c207072696360448201527f65206d7573742062652067726561746572207468616e203000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603c60248201527f6661696c656420746f2076616c69646174653a20657374696d6174656420676160448201527f7320666565206d7573742062652067726561746572207468616e2030000000006064820152fd5b5048156154d1565b73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300541633036157e557565b60246040517f118cdaa7000000000000000000000000000000000000000000000000000000008152336004820152fd5b818110615820575050565b5f8155600101615815565b9081518110156143e2570160200190565b60ff7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a005460401c161561586b57565b60046040517fd7e6bcf8000000000000000000000000000000000000000000000000000000008152fd5b906158d457508051156158aa57805190602001fd5b60046040517fd6bda275000000000000000000000000000000000000000000000000000000008152fd5b8151158061592c575b6158e5575090565b60249073ffffffffffffffffffffffffffffffffffffffff604051917f9996b315000000000000000000000000000000000000000000000000000000008352166004820152fd5b50803b156158dd565b7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8111615c4057610100907f80000000000000000000000000000000000000000000000000000000000000008114611f8957805f031680615c37575b6fffffffffffffffffffffffffffffffff8116615c06575b77ffffffffffffffff0000000000000000ffffffffffffffff8116615bd5575b7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff8116615ba4575b7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff8116615b73575b7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff8116615b42575b7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f8116615b11575b7f33333333333333333333333333333333333333333333333333333333333333338116615ae0575b7f555555555555555555555555555555555555555555555555555555555555555516615ab35790565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8101908111611f895790565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790615a8a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8101908111611f895790615a62565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff88101908111611f895790615a3a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff08101908111611f895790615a13565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101908111611f8957906159ed565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc08101908111611f8957906159c9565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808101908111611f8957906159a9565b60ff9150615991565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f496e7075742065786365656473206d6178696d756d20696e743235362076616c60448201527f75650000000000000000000000000000000000000000000000000000000000006064820152fd5b5f5260205260205f60408160025afa156136ae577fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f5f51169056fea2646970667358221220a2d6c54c44a68038f7d59aa211cb3b39a5f231a0e7b9f3f652141521dc58b6fd64736f6c63430008170033","sourceMap":"2048:37005:43:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2267:2;2048:37005;2267:2;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;8837:64:24;2048:37005:43;;;;;;;4301:16:24;2048:37005:43;;;;4726:16:24;;:34;;;;2048:37005:43;;4790:16:24;:50;;;;2048:37005:43;4855:13:24;:30;;;;2048:37005:43;4851:91:24;;;2048:37005:43;;;;;;;;4979:67:24;;2048:37005:43;6893:76:24;;;:::i;:::-;;;:::i;:::-;6961:1;7731:10:43;6961:1:24;:::i;:::-;6893:76;;:::i;:::-;2048:37005:43;;;;5066:101:24;;2048:37005:43;;;5066:101:24;2048:37005:43;;;;;;5142:14:24;2048:37005:43;;;;;;5142:14:24;2048:37005:43;;4979:67:24;2048:37005:43;;;;;;4979:67:24;;;4851:91;2048:37005:43;;;4908:23:24;;;;4855:30;4872:13;;;4855:30;;;4790:50;4818:4;4810:25;:30;;-1:-1:-1;4790:50:24;;4726:34;;;-1:-1:-1;4726:34:24;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;23384:9;;;;2048:37005;;;;23466:15;2048:37005;;;;;;;;;;23452:10;:36;2048:37005;;23557:11;;2048:37005;;;;;23633:18;2048:37005;;;;;;23680:12;;:30;2048:37005;;23752:40;2048:37005;;23889:47;;;:::i;:::-;2048:37005;;;;23633:18;2048:37005;;29356:40;2048:37005;;;;29356:40;:::i;:::-;2048:37005;;;24030:14;2048:37005;;;;;;;24103:11;2048:37005;;24092:30;2048:37005;;;;24092:30;:::i;:::-;2048:37005;24086:3;2048:37005;;;;;;24141:12;;;;;;;;;24136:1885;24168:3;2048:37005;;;;24155:11;;;;2048:37005;;25323:32;2048:37005;25323:32;;2048:37005;;;;;;;;;;;;;;;25323:32;;;2048:37005;;;;;;;;;;;;;;;;;;;;;25323:32;;25404:18;2048:37005;;;;25570:47;2048:37005;;;24086:3;2048:37005;;;25570:47;;:::i;:::-;25554:63;2048:37005;;;25554:63;;:::i;:::-;;;2048:37005;;;25554:63;;:::i;:::-;;25674:41;25693:13;2048:37005;;;25693:13;;:::i;:::-;;2048:37005;25674:41;;:::i;:::-;2048:37005;370:8:40;;2048:37005:43;370:21:40;2048:37005:43;;;;;;;;:::i;:::-;;;;;;;;;;480:10:40;492:6;2048:37005:43;492:6:40;;;;595:18;;;;;2048:37005:43;;;;;;;;;;;;;475:104:40;2048:37005:43;;;;25773:15;2048:37005;;;;;25796:13;2048:37005;;;25796:13;;:::i;:::-;;2048:37005;;;;;;;;;;;;;;;;;25762:61;;;:::i;:::-;24086:3;2048:37005;;24086:3;2048:37005;;;;25821:1;2048:37005;;;;;;;;;;25919:13;2048:37005;25903:9;2048:37005;25875:15;:9;2048:37005;;;;;25875:9;;:::i;:::-;:15;;;;;:::i;:::-;;;;2048:37005;;25903:9;;;:::i;:::-;2048:37005;;;25919:13;;:::i;:::-;;:20;2048:37005;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;24103:11;2048:37005;;;;;;;;;;;;24103:11;2048:37005;;;;;;;;1260:12:44;;;;;2048:37005:43;1260:30:44;2048:37005:43;;1351:50:44;;1878:13;;1351:50;;1878:13;;1911:3;1897:12;;2048:37005:43;1893:16:44;;;;;25821:1:43;2042:8:44;;;;;;:::i;:::-;2048:37005:43;;;;;25821:1;;10374:22:44;;;;:::i;:::-;2064:207;;1025:5:42;;1911:3:44;2048:37005:43;1878:13:44;;;;2064:207;10374:22;;;:::i;:::-;2064:207;;;1893:16;;;;;;;;;;;;;;;1351:58;;;2048:37005:43;;24168:3;;;:::i;:::-;24141:12;;;;;;;;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;25773:15;2048:37005;;;;;;;;;;500:3:40;534:8;;2048:37005:43;;;25323:32;2048:37005;;;;;534:34:40;2048:37005:43;;543:24:40;25821:1:43;2048:37005;25323:32;2048:37005;;;543:24:40;:::i;:::-;534:34;;:::i;:::-;2048:37005:43;;519:49:40;;;;;;:::i;:::-;;2048:37005:43;480:10:40;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;24155:11;;;;;;26425:22;24155:11;;26438:9;26425:22;;:::i;:::-;28560:24;28599:13;;28594:217;28614:17;;;;;;2048:37005;;;;;;;;;;26491:4;2048:37005;;;;;;;;;;;;;;;27871:23;26424:72;;;;:::i;:::-;27881:13;27871:23;;:::i;:::-;2048:37005;;;24030:14;2048:37005;;;;;;;;;;;;;;28050:203;28199:44;28004:16;;;2048:37005;28004:16;;;;:::i;:::-;2048:37005;;;;;;;;28214:22;2048:37005;;;;23680:12;28199:44;:::i;:::-;2048:37005;24103:11;2048:37005;28050:203;;:::i;:::-;8244:9;;:19;2048:37005;;;8317:34;;;;2183:42;8317:34;;;;:::i;:::-;;2048:37005;;;;28295:59;2048:37005;28372:20;2048:37005;;;;;;;;;;;;;;;25773:15;2048:37005;;;;;28295:59;8244:9;28372:20;:::i;:::-;2048:37005;;;;26611:15;2048:37005;;;;;;;26651:26;;26647:160;;28594:217;2048:37005;;;26886:35;2048:37005;;26886:35;2048:37005;;28214:22;2048:37005;;23680:12;2048:37005;;;;;;26886:35;;;;;:::i;:::-;;;;27131:10;;27127:144;;2048:37005;;27127:144;23452:10;;;27224:36;23452:10;;27176:34;;;;:::i;:::-;;27224:36;:::i;:::-;27127:144;;2048:37005;;26647:160;26747:16;2048:37005;;;;;;26697:95;;;;;;2048:37005;;;;;;;26697:95;;;;;2048:37005;26697:95;;;2048:37005;26697:95;;2048:37005;;;;;;;;;;;;;26697:95;;;;;;;;26647:160;;;;;26697:95;;;;:::i;:::-;2048:37005;;26697:95;;;;2048:37005;;;;26697:95;2048:37005;;;;;;;;;26697:95;2048:37005;;;;;;;;;;;;;;;23466:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;28633:3;28772:9;;:15;:9;;;;;:::i;:::-;2048:37005;28772:15;;;;:::i;:::-;2048:37005;;;24103:11;2048:37005;;;;;;;;;;;;;;;;;;;;;25821:1;28750:50;;;;:::i;:::-;28633:3;2048:37005;28599:13;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;23466:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2357:1:23;2048:37005:43;;:::i;:::-;2303:62:23;;:::i;:::-;2357:1;:::i;:::-;2048:37005:43;;;;:::i;:::-;;;;;;;;:::i;:::-;16941:18;;16933:48;16941:18;;;:::i;:::-;16933:48;:::i;:::-;2048:37005;;;16999:30;2048:37005;;;;;;;;;;17040:10;16999:51;2048:37005;;;;;;;17160:15;2048:37005;;;;;;;;;;;;;17040:10;;;2048:37005;;;;;16999:30;2048:37005;;;;;;;;;;;;;17040:10;;17296:61;;;;;;17390:15;2048:37005;;;;;;;17426:26;;17422:155;;2048:37005;;;17422:155;17468:98;;;;;;2048:37005;17468:98;;;;;;;;;;2048:37005;17468:98;;2048:37005;17468:98;;2048:37005;;;;;17040:10;2048:37005;;;;;;;;;;;;;;:::i;:::-;17468:98;;;;;;;;;;17422:155;;;;;;2048:37005;;;17468:98;;;;:::i;:::-;2048:37005;;17468:98;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;20193:72;2529:4;20201:39;;;;;;;20193:72;:::i;:::-;20275:48;20283:18;;;:::i;20275:48::-;20341:11;;2048:37005;;;;;20402:15;2048:37005;;;;;;;;20428:10;20402:36;2048:37005;;;;;20514:11;2048:37005;;;;;;20570:31;;;;;:::i;:::-;20618:13;20633:11;;;;;;2048:37005;;20795:28;2048:37005;;;;;;;;;;;;;;:::i;:::-;20795:28;;;2048:37005;;;20857:15;2048:37005;;;;;;;;20893:26;;20889:135;;20613:168;2048:37005;;;;;;;;20889:135;20935:78;;;;;2048:37005;;;;;;;;;;20935:78;;;2048:37005;20935:78;;2048:37005;;;20935:78;2048:37005;20935:78;;2048:37005;;;;;;;;;;;;;;;;;;20514:11;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;20935:78;;;;;;;;;;2048:37005;20889:135;;;;;;;20935:78;;;;;:::i;:::-;2048:37005;;20935:78;;;2048:37005;;;;20935:78;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;20935:78;2048:37005;;;20646:3;20687:12;;;;;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;20707:12;;;;;:::i;:::-;:20;2048:37005;;;;;21246:116;;21375:12;;21371:102;;2316:7;21486:24;;21482:120;;2048:37005;;;;;;;;20514:11;2048:37005;;;;;;;;21679:20;;;;:::i;:::-;2048:37005;;38022:21;38033:9;;;:::i;:::-;38022:21;:::i;:::-;1025:5:42;;20514:11:43;1025:5:42;35062:13:43;;35057:129;35077:5;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;21756:9;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;35057:129;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;1025:5:42;20514:11:43;1025:5:42;2048:37005:43;;;;;;;21852:16;2048:37005;;21852:36;2048:37005;;;1025:5:42;20514:11:43;1025:5:42;2048:37005:43;;21852:36;:::i;:::-;2048:37005;;20756:14;;;;:::i;:::-;20742:28;;;;:::i;:::-;2048:37005;;20618:13;;;;;2048:37005;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;20514:11;2048:37005;;;;-1:-1:-1;2048:37005:43;;;;;20514:11;2048:37005;;;;:::i;:::-;;;;;;;-1:-1:-1;2048:37005:43;;;;35084:3;2048:37005;;;;;;;;;35145:30;2048:37005;;35115:16;2048:37005;;;;35115:16;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;35145:30;;:::i;:::-;35084:3;2048:37005;35062:13;;;;;;;;;;;21482:120;2048:37005;;;;21533:58;;;;2048:37005;21533:58;;2048:37005;;;;;;;;;;;;;;;;;;;;;21533:58;21371:102;2048:37005;;;;21410:52;;;;2048:37005;21410:52;;2048:37005;;;;;;;;;;;;;;;;21410:52;21246:116;2048:37005;;;;21297:54;;;;2048:37005;21297:54;;2048:37005;;;;;;;;;;;;;;;;21297:54;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;9446:11;2048:37005;;9435:30;2048:37005;;;;9435:30;:::i;:::-;9429:3;2048:37005;;;;;;;;;;;9537:14;2048:37005;;;;;;;;;;;;;;;;;9515:51;;;;;:::i;:::-;9584:10;;;;2048:37005;;;;;9598:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;9584:53;;2048:37005;;9724:25;;;;:::i;:::-;:51;;;;2048:37005;;;;;;;;;;;9724:51;2048:37005;-1:-1:-1;9753:22:43;;-1:-1:-1;9724:51:43;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;2267:2;2048:37005;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;:::i;:::-;17890:39;;17882:72;2529:4;17890:39;;;17882:72;:::i;:::-;1025:5:42;18019:9:43;;;:21;2048:37005;;;8317:34;;;;2183:42;8317:34;;;;:::i;:::-;;2048:37005;;;235:1:42;2048:37005:43;;;;;18116:15;;;;:::i;:::-;2048:37005;;;;235:1:42;2048:37005:43;;;;;;18141:16;2048:37005;;;;;;;18178:18;2048:37005;;;;;;;18289:15;2048:37005;;;;;;;18314:10;;2048:37005;;;;;;;18334:15;2048:37005;;;;;;;;;;;;;;;;18381:22;2048:37005;;;;;;;18443:26;18439:127;;2048:37005;18314:10;;;;18580:33;;;;;18019:9;18703:20;18699:168;;2048:37005;;;;;;;;18699:168;2048:37005;18019:9;2048:37005;18019:9;;2048:37005;;;;18314:10;;;18820:36;18314:10;;18758:48;;;;:::i;18820:36::-;18699:168;;;2048:37005;;;;;;;;;;18439:127;18485:70;;;;;2048:37005;;;;;;;;18485:70;;;;;;2048:37005;18485:70;;;2048:37005;18485:70;;2048:37005;18314:10;2048:37005;;;;;;;;;;;;;;:::i;:::-;18485:70;;;;;;;;;;;;;18439:127;;;;18485:70;;;;:::i;:::-;2048:37005;;18485:70;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;9916:48;9924:18;;;:::i;9916:48::-;2048:37005;;9981:16;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2377:4;2048:37005;;;;;;;;;;;;;;;2303:62:23;;:::i;:::-;8837:64:24;2048:37005:43;;;;;;;6431:44:24;;;;2048:37005:43;6427:105:24;;2048:37005:43;8004:1;2048:37005;8022:59;2048:37005;;;:::i;:::-;;811:66:30;2048:37005:43;;;;;;;;;;;;;;;:::i;:::-;;;;;;8022:59;;;2048:37005;;;;6656:20:24;2048:37005:43;;;8004:1;2048:37005;;6656:20:24;2048:37005:43;;6431:44:24;2048:37005:43;8004:1;2048:37005;;;6450:25:24;;6431:44;;2048:37005:43;;;;;;;;;;;;;;1280:65:23;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;2316:7;2048:37005;;;;;;;;;;;;;;;;;;;;11982:48;11990:18;;;:::i;11982:48::-;2048:37005;;12047:14;2048:37005;;;;;;;;;;;;;;;;;;:::i;:::-;19057:39;19049:72;2529:4;19057:39;;;19049:72;:::i;:::-;2048:37005;19144:13;2048:37005;;19135:22;;;19131:88;;2048:37005;;;;;19237:15;2048:37005;;;;;;;;;;19263:10;19237:36;2048:37005;;;;;;;;19361:16;2048:37005;;;;;;;;;;;19237:15;2048:37005;;;;;;;;;;;19476:18;2048:37005;;;;;;;19515:22;2048:37005;;;;;;;19596:15;2048:37005;;;;;;;19632:26;;19628:133;;2048:37005;;;;;19775:39;2048:37005;;;;;;19775:39;2048:37005;;19628:133;19674:76;;;;;2048:37005;;;;;;;19674:76;;;;;;2048:37005;19674:76;;;2048:37005;19674:76;;2048:37005;;;;;;;;;;;;;;;;:::i;:::-;19674:76;;;;;;;;;;19628:133;;;;;19674:76;;;;:::i;:::-;2048:37005;;19674:76;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;19131:88;2048:37005;;;19173:35;;;2048:37005;;19173:35;;2048:37005;;;;;;;;;;;19173:35;2048:37005;;;;;;;;;;;;2303:62:23;;:::i;:::-;2048:37005:43;;1280:65:23;2048:37005:43;;;;;;;;;3975:40:23;;;;2048:37005:43;;;;;;;;;;;;;;;;12280:48;12288:18;;;:::i;12280:48::-;2048:37005;;12367:17;2048:37005;;;;;;;12427:30;;;;:::i;:::-;12472:13;12487:19;;;;;;2048:37005;;;;;;;;;;;;;;:::i;12508:3::-;12539:11;;2048:37005;12539:11;;;:::i;:::-;2048:37005;;;;;;12527:23;;;;:::i;:::-;2048:37005;;12472:13;;2048:37005;;;;;;;;;;;;;;;;;10381:48;10389:18;;;:::i;10381:48::-;2048:37005;;10446:18;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2626:42;2048:37005;;;;;;;;;;;;;;;;;;2580:5;2048:37005;;;;;;;;;;;;;;;;;;;;12871:19;12910:18;12902:48;12910:18;;;:::i;12902:48::-;2048:37005;;;12982:11;2048:37005;;;;;;13015:13;;13030:14;;;;;;2048:37005;;;;;;;;13046:3;2048:37005;;;13069:15;2048:37005;;;;;;;;;;;;;;13065:81;;13046:3;2048:37005;;13015:13;;13065:81;13118:13;;2048:37005;13118:13;;:::i;:::-;13065:81;;;;;2048:37005;;;;;;;;;;;;;5115:6:25;2048:37005:43;5106:4:25;5098:23;5094:145;;2048:37005:43;;;811:66:30;2048:37005:43;;;5094:145:25;2048:37005:43;;;5199:29:25;;;;2048:37005:43;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4692:6:25;;2048:37005:43;4683:4:25;;4675:23;:120;;;;;2048:37005:43;4658:251:25;;;2303:62:23;;:::i;:::-;2048:37005:43;;;;;6156:52:25;2048:37005:43;6156:52:25;;;;2048:37005:43;6156:52:25;;;;;;;;;;;2048:37005:43;-1:-1:-1;;6152:437:25;;2048:37005:43;;;;6518:60:25;;;;2048:37005:43;6518:60:25;;2048:37005:43;6518:60:25;6152:437;811:66:30;;;;6250:40:25;;;;6246:120;;1748:29:30;;;:34;1744:119;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;2407:36:30;2048:37005:43;;2407:36:30;2048:37005:43;;2458:15:30;:11;;4049:25:33;4091:55;4049:25;;;;;;;;;;:::i;:::-;4091:55;;:::i;:::-;;2048:37005:43;;2454:148:30;6163:9;;;;;;6159:70;;2454:148;2048:37005:43;;6159:70:30;6199:19;;2048:37005:43;6199:19:30;;;1744:119;2048:37005:43;;;1805:47:30;;;;2048:37005:43;1805:47:30;;2048:37005:43;1805:47:30;6246:120:25;2048:37005:43;;6317:34:25;;;;2048:37005:43;6317:34:25;;2048:37005:43;6317:34:25;6156:52;;;;;;;;;;;;;;;;;:::i;:::-;;;2048:37005:43;;;;;6156:52:25;;;;;2048:37005:43;;;;6156:52:25;;;;;4675:120;2048:37005:43;;;811:66:30;2048:37005:43;;4753:42:25;;4675:120;;;2048:37005:43;;;;;;;;:::i;:::-;;;;27408:14;2048:37005;;;;;;;;;;;;;;27485:16;2048:37005;27485:16;27519:203;27485:16;;27668:44;2048:37005;27485:16;;:::i;:::-;2048:37005;;;;;27683:22;2048:37005;;;;27668:12;:44;:::i;27519:203::-;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;30423:39;30415:72;2529:4;30423:39;;;30415:72;:::i;:::-;2048:37005;;;;30519:15;2048:37005;;;;;;;;30505:10;:36;2048:37005;;;;;;30621:16;2048:37005;;;;;;30621:27;2048:37005;;;;;;30712:22;2048:37005;;;;;;;30712:48;30708:123;;2048:37005;;30920:17;2048:37005;;;;;;;31003:13;30999:387;;2048:37005;;;;;;;30621:16;2048:37005;;;;;;31443:14;2048:37005;;;;;;31521:32;2048:37005;;31521:12;:32;:::i;:::-;2048:37005;;31504:49;31500:161;;2048:37005;;;;;;31670:18;2048:37005;;;;;;;;30621:16;2048:37005;;;;;;31876:28;31872:208;;2048:37005;32113:15;2048:37005;;;;;;;;32149:26;;32145:170;;2048:37005;;;;;;30621:16;2048:37005;;;;;;;;;;;;;;;;;32329:65;2048:37005;;;32329:65;;2048:37005;;32145:170;31670:18;2048:37005;;;;;;;30621:16;2048:37005;;;;;;32191:113;;;;;2048:37005;;;;;;32191:113;;;;;;;2048:37005;32191:113;;2048:37005;;;32191:113;;2048:37005;;;;;;;;;;;;;;;;;;;:::i;:::-;32191:113;;;;;;;;;;32145:170;;;;;32191:113;;;;:::i;:::-;2048:37005;;32191:113;;;;31872:208;2048:37005;;31925:19;;;;30712:22;2048:37005;;;;;;;31670:18;2048:37005;;;;;;;31872:208;;31500:161;2048:37005;;;31569:81;;;2048:37005;;31569:81;;2048:37005;;;;;;;;;;;;;;;;;;;;;31569:81;30999:387;31069:24;;;:::i;:::-;31113:13;;31128;;;;;;2048:37005;;;;;32537:48;32545:18;2048:37005;;32545:18;:::i;32537:48::-;32595:22;;32668:3;2048:37005;;32647:19;;;;;32722:11;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;;;;;;33152:5;;2048:37005;;;;;35407:11;2048:37005;;35396:30;2048:37005;;;;35396:30;:::i;:::-;2048:37005;35390:3;2048:37005;;;;;;38033:9;38022:21;38033:9;;;:::i;38022:21::-;2048:37005;35390:3;2048:37005;35644:8;;;:38;;;35637:177;35644:38;;;2048:37005;;35748:15;2048:37005;;;;;;;;;;;;;;;;;;;;35698:36;2048:37005;;;35698:36;:::i;:::-;2048:37005;;;35748:15;;:::i;:::-;38033:9;38022:21;38033:9;;;:::i;38022:21::-;35637:177;;;;;35644:38;;;;;;;;2048:37005;35644:38;32686:48;35644:38;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;33223:9;2048:37005;;;;;;;;;;;;;;;;;;:::i;:::-;;;;35637:177;32686:48;;;;:::i;:::-;32668:3;2048:37005;32632:13;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;35407:11;2048:37005;;;;;;;:::i;:::-;;;;;35644:38;2048:37005;;;;;35407:11;2048:37005;;;;;;35656:26;;35644:38;;2048:37005;;;;;;;;;;32647:19;;;;;;;2048:37005;;;;30621:16;2048:37005;;32754:37;2048:37005;;;;;;32754:37;:::i;:::-;2048:37005;;31336:39;2048:37005;;;;;;;;;;;;;;;:::i;:::-;31336:39;;;30999:387;;;;31143:3;2048:37005;;;;;;;;;;;;31189:29;;;;:::i;:::-;2048:37005;;;;;;;;31166:52;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;31113:13;;2048:37005;;;;;;;;;;30708:123;30808:12;2048:37005;;30708:123;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;8676:13;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;16262:48;16270:18;;;:::i;16262:48::-;2048:37005;;;16353:15;2048:37005;;;;;;;16419:10;16393:36;;2048:37005;;16516:44;;2048:37005;;;;;16689:30;2048:37005;;;;;;;;;;;;;16512:314;2048:37005;;;16757:30;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;37518:11;2048:37005;;37507:30;2048:37005;;;;37507:30;:::i;:::-;37501:3;;2048:37005;;;;;;;;37592:51;;;:::i;:::-;37658:13;37673:21;;;;;;2048:37005;;;;;;;:::i;37696:3::-;37749:13;37727:41;37749:13;;2048:37005;37749:13;;;;:::i;:::-;2048:37005;37727:41;;:::i;:::-;37715:53;;;;:::i;:::-;;;;;;:::i;:::-;;2048:37005;37658:13;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;10618:48;10626:18;;;:::i;10618:48::-;2048:37005;;10683:15;2048:37005;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;10908:48;10916:18;;;:::i;10908:48::-;2048:37005;;10974:15;2048:37005;;;;;;;;;;;10998:30;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;10144:48;10152:18;;;:::i;10144:48::-;2048:37005;;10209:11;2048:37005;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;:::i;:::-;;;;;;;;;;;;;;;2779:66;2048:37005;;;;;;;;;;;;;;;;;;2435:42;2048:37005;;;;;;;;;;;;;;:::i;:::-;11690:18;11682:48;11690:18;;;:::i;11682:48::-;2048:37005;;11747:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;22230:39;22222:72;2529:4;22230:39;;;;;;22222:72;:::i;:::-;22304:48;22312:18;;;:::i;22304:48::-;2048:37005;;;;22370:15;2048:37005;;;;;;;;;;22396:10;22370:36;2048:37005;;;;;22503:17;;2048:37005;;2377:4;22485:49;2048:37005;;;;22485:49;;:::i;:::-;:74;2048:37005;;;22653:19;;;;;;2048:37005;;22881:15;2048:37005;;;;;;;;;;22917:26;;22913:132;;2048:37005;;;22913:132;22959:75;;;;;2048:37005;;22959:75;2048:37005;22959:75;;2048:37005;22959:75;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;22714:11;2048:37005;;;;;;;;;;;;;;;;;;;;;:::i;:::-;22959:75;;;;;;;;;;22913:132;;;;;2048:37005;;;22959:75;;;;;:::i;:::-;2048:37005;22959:75;;;;2048:37005;;;;;;;;;22674:3;22700:11;;;;;:::i;:::-;2048:37005;;;;22714:11;2048:37005;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;22825:11;;;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;22638:13;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2183:42;2048:37005;;;;;;;;;;;;;;;;11139:48;11147:18;;;:::i;11139:48::-;2048:37005;;11204:22;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2529:4;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;:::i;:::-;;;:::o;:::-;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;31216:1;2048:37005;;;;;;;:::o;:::-;;;;;;;;;;:::o;:::-;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;:::o;:::-;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;:::o;9027:191::-;;9122:18;;;:::i;:::-;:50;;;;9027:191;9122:89;;;9115:96;;9027:191;:::o;9122:89::-;2048:37005;;;-1:-1:-1;2048:37005:43;9176:15;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;9176:35;;9027:191;:::o;9122:50::-;2048:37005;;;-1:-1:-1;2048:37005:43;9154:11;2048:37005;;;-1:-1:-1;2048:37005:43;;9144:28;;9122:50;;;2048:37005;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;11309:198;2048:37005;;;;;;:::i;:::-;;11410:48;11418:18;;;:::i;11410:48::-;-1:-1:-1;2048:37005:43;;;11475:9;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;;;;;;:::i;:::-;;;;;-1:-1:-1;2048:37005:43;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;11309:198;:::o;2048:37005::-;-1:-1:-1;2048:37005:43;;;;;;;;-1:-1:-1;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;:::o;:::-;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;-1:-1:-1;2048:37005:43;;;;;;;;;;:::o;:::-;;;;;;;;:::i;:::-;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;-1:-1:-1;2048:37005:43;;;;;;;;;;:::o;:::-;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;:::o;:::-;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;:::o;13671:2339::-;;2048:37005;13935:48;13943:18;;;:::i;13935:48::-;14001:9;;2048:37005;;;;;14131:11;2048:37005;;;;;;14237:21;;;;:::i;:::-;14300:20;;;;:::i;:::-;14362;;;;:::i;:::-;14393:23;2048:37005;14426:23;2048:37005;14465:13;2048:37005;14480:14;;;;;;14460:665;15200:16;;;;;;;15196:808;15200:16;;;2048:37005;;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;15262:69;;;2048:37005;15262:69;:::o;15196:808::-;15352:19;;;;;;;;15348:656;15352:19;;;15464:27;;;:::i;:::-;15516:26;;;;:::i;:::-;15567;;;;:::i;:::-;15613:13;2048:37005;15628:15;;;;;;15348:656;;;;;13671:2339::o;15645:3::-;15680:13;;2048:37005;15680:13;;;:::i;:::-;;15668:25;;;;:::i;:::-;;;;;;:::i;:::-;;15725:15;;;;:::i;:::-;2048:37005;15711:29;;;;:::i;:::-;2048:37005;15772:15;;;;:::i;:::-;2048:37005;15758:29;;;;:::i;:::-;2048:37005;;15613:13;;15348:656;15900:19;;;15348:656;13671:2339::o;14496:3::-;2048:37005;;;14519:15;2048:37005;;;;;;;;;;;;;;14515:600;;14496:3;2048:37005;;14465:13;;14515:600;14572:21;;;;;;:44;;;14515:600;14568:502;;;2048:37005;;;14666:9;2048:37005;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;14640:45;;;;:::i;:::-;;;;;;:::i;:::-;;14707:29;;;;;:::i;:::-;2048:37005;;;;14519:15;2048:37005;;;;;;;;;;;;;;;;14131:11;2048:37005;;;;;;;;;;;;;14838:13;14758:58;2048:37005;14758:58;;;;15087:13;14758:58;;:::i;:::-;2048:37005;14838:13;:::i;:::-;14568:502;15087:13;:::i;:::-;14515:600;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;14131:11;2048:37005;;;;;;;14568:502;14895:14;;;;;:::i;:::-;14880:29;;;14876:194;;15087:13;2048:37005;14568:502;15087:13;:::i;14876:194::-;15010:14;;;;;;;2048:37005;15046:5;;;;;;;;14572:44;14597:19;;;;14572:44;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;:::i;:::-;;;;-1:-1:-1;2048:37005:43;;;;:::o;:::-;;;:::o;28852:406::-;-1:-1:-1;28852:406:43;;2048:37005;;29032:23;;;2048:37005;;;29032:23;;;;;;:::i;:::-;28999:57;;2435:42;28999:57;;;;:::i;:::-;2048:37005;;;;29032:23;2048:37005;;;29222:29;;2048:37005;;;;29032:23;29222:29;2048:37005;28852:406;:::o;2048:37005::-;;;;;;;29032:23;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;38142:909;2048:37005;;;;38262:97;2048:37005;38262:97;;2779:66;38262:97;;;;;2048:37005;2580:5;2048:37005;;;;38262:97;2626:42;;38262:97;;;2048:37005;38262:97;;;;-1:-1:-1;;38262:97:43;;;38142:909;-1:-1:-1;38258:787:43;;38645:400;;;38758:26;2048:37005;38645:400;;:::i;:::-;2048:37005;;;;;;;;;;;;;:::i;:::-;38758:26;;;2048:37005;;;38836:42;;;;2048:37005;38836:42;;38262:97;38836:42;;2048:37005;38836:42;;;;;;;-1:-1:-1;38836:42:43;;;38258:787;2048:37005;;38892:80;-1:-1:-1;2048:37005:43;;;;38900:19;38892:80;:::i;:::-;2048:37005;;;;39019:14;;2048:37005;;;38986:48;:::o;38836:42::-;;;;;;;-1:-1:-1;38836:42:43;;;;;;;:::i;:::-;;;;;:::i;:::-;;;;;;;;;;;;2048:37005;;;-1:-1:-1;2048:37005:43;;;;;38258:787;2048:37005;;;38419:80;2048:37005;;-1:-1:-1;2048:37005:43;;;;;38427:19;38419:80;:::i;38262:97::-;;;;;;;;;;;;;;;:::i;:::-;;;;;2048:37005;;;;;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;8772:148;2048:37005;8860:13;2048:37005;;8852:21;;:61;;;;8845:68;8772:148;:::o;8852:61::-;2048:37005;;-1:-1:-1;2048:37005:43;8877:15;2048:37005;;;;-1:-1:-1;2048:37005:43;;;8877:36;;8772:148;:::o;2048:37005::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;3405:215:23:-;2048:37005:43;;;;3489:22:23;;;3485:91;;1280:65;2048:37005:43;;;;;;;;;;;3975:40:23;-1:-1:-1;3975:40:23;;3405:215::o;3485:91::-;2048:37005:43;;;3534:31:23;;;3509:1;3534:31;;;2048:37005:43;3534:31:23;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;196:563:39:-;274:3;2048:37005:43;;316:3:39;2048:37005:43;325:6:39;321:32;;196:563;2048:37005:43;;371:2:39;2048:37005:43;380:6:39;376:32;;196:563;2048:37005:43;;426:2:39;2048:37005:43;435:6:39;431:32;;196:563;2048:37005:43;;481:2:39;2048:37005:43;490:6:39;486:32;;196:563;2048:37005:43;;536:1:39;2048:37005:43;545:6:39;541:32;;196:563;2048:37005:43;;591:1:39;2048:37005:43;600:6:39;596:32;;196:563;2048:37005:43;;646:1:39;2048:37005:43;655:6:39;651:32;;196:563;2048:37005:43;;701:1:39;2048:37005:43;706:24:39;;747:5;;;:::i;:::-;196:563;:::o;706:24::-;2048:37005:43;;;;;;;;;718:12:39;:::o;651:32::-;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;675:5:39;651:32;;;596;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;620:5:39;596:32;;;541;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;565:5:39;541:32;;;486;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;510:5:39;486:32;;;431;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;455:5:39;431:32;;;376;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;400:5:39;376:32;;;321;335:8;;;316:3;345:5;321:32;;;35854:1341:43;;2048:37005;;;;;;;;:::i;:::-;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;;;-1:-1:-1;2048:37005:43;36013:16;2048:37005;;;-1:-1:-1;2048:37005:43;;36001:35;;2048:37005;;;36096:1;;2048:37005;;;;;;;;;;;;36075:34;;-1:-1:-1;36271:15:43;36266:617;;36288:5;;;2048:37005;;;-1:-1:-1;2048:37005:43;36571:13;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;36904:37;2048:37005;-1:-1:-1;2048:37005:43;;36904:37;;:::i;:::-;36955:22;;;;36951:152;;37172:15;;;;:::i;:::-;2048:37005;;;;;;:::i;:::-;;;37134:54;;2048:37005;35854:1341;:::o;36951:152::-;2048:37005;;36096:1;2048:37005;;;;;;;37070:21;;;:::i;36295:3::-;2048:37005;;;;-1:-1:-1;2048:37005:43;36435:11;2048:37005;;;-1:-1:-1;2048:37005:43;;36422:31;;;36418:121;;2048:37005;-1:-1:-1;2048:37005:43;36571:13;2048:37005;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;36565:37;2048:37005;-1:-1:-1;2048:37005:43;;36565:37;;:::i;:::-;36644:22;;;36686:38;2048:37005;;-1:-1:-1;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;36686:38;;:::i;:::-;2048:37005;;;;;;;;;;;;;36742:25;2048:37005;;36742:25;;:::i;:::-;36640:233;;2048:37005;;;;;;36271:15;;;;;2048:37005;;;;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;36640:233;2048:37005;;;;;;;;;;;;;36833:25;2048:37005;;36833:25;;:::i;:::-;36640:233;;;36418:121;2048:37005;;;;;;;;;;36473:25;2048:37005;;36473:25;;:::i;2048:37005::-;;;;;;;;;;;;;;;;;;;;;;;;277:15:42;;;;;;;;;;;:::o;1669:1834::-;;;;;1927:19;;;;:41;;;1669:1834;2048:37005:43;;;;;2051:15:42;;;2048:37005:43;;2145:11:42;;2048:37005:43;;;;;;1945:1:42;2326:20;;1945:1;;2467:45;2516:29;2467:45;:79;:45;;:::i;:::-;277:15;;;2516:29;:::i;:::-;2467:79;;:::i;:::-;1025:5;;;;2874:46;1025:5;2874:36;1025:5;2048:37005:43;1025:5:42;2874:36;:::i;:::-;:46;:::i;:::-;2048:37005:43;799:1:42;2048:37005:43;;;;;;;;799:1:42;2048:37005:43;;;;;;3043:3:42;1025:5;;2048:37005:43;;;;;;;689:1:42;2048:37005:43;;;;;;3152:32:42;;;;;3200:8;;;;;1945:1;3200:8;:::o;3148:349::-;3043:3;1025:5;;3277:31;;3043:3;;3331:31;;;;;:::i;3273:224::-;2048:37005:43;;;;;;235:1:42;2048:37005:43;;;;;3043:3:42;1025:5;;3426:60;:::o;2322:452::-;277:15;;;;;;;2665:30;277:15;1945:1;277:15;;;2665:30;:::i;:::-;2048:37005:43;;;;;;;;;;;;2717:45:42;;;:::i;:::-;1025:5;;;;;2874:46;1025:5;2874:36;1025:5;;2874:36;:::i;2048:37005:43:-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;1927:41:42;1950:13;;:18;1927:41;;2658:162:23;2048:37005:43;1280:65:23;2048:37005:43;;966:10:26;2717:23:23;2713:101;;2658:162::o;2713:101::-;2048:37005:43;;;2763:40:23;;;966:10:26;2763:40:23;;;2048:37005:43;2763:40:23;2048:37005:43;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;:::o;7084:141:24:-;2048:37005:43;8837:64:24;2048:37005:43;;;;7150:18:24;7146:73;;7084:141::o;7146:73::-;7191:17;2048:37005:43;;7191:17:24;;;;4421:582:33;;4593:8;;-1:-1:-1;2048:37005:43;;5674:21:33;:17;;5799:158;;;;;;5670:354;5994:19;2048:37005:43;;5994:19:33;;;;4589:408;2048:37005:43;;4841:22:33;:49;;;4589:408;4837:119;;4969:17;;:::o;4837:119::-;2048:37005:43;;;;;4917:24:33;;;;2048:37005:43;4917:24:33;;;2048:37005:43;4917:24:33;4841:49;4867:18;;;:23;4841:49;;1613:793:39;1700:16;1687:30;;2048:37005:43;;1778:3:39;2048:37005:43;;;;;;;-1:-1:-1;2048:37005:43;1828:13:39;1855:6;1851:40;;1613:793;791:66;1904:11;;1900:55;;1613:793;888:66;1968:10;;1964:53;;1613:793;985:66;2030:10;;2026:53;;1613:793;1082:66;2092:10;;2088:53;;1613:793;1178:66;2154:9;;2150:51;;1613:793;1274:66;2214:9;;2210:51;;1613:793;1370:66;2274:9;;2270:51;;1613:793;1466:66;2334:9;2330:51;;1613:793;:::o;2330:51::-;2048:37005:43;;;;;;;;1613:793:39;:::o;2270:51::-;2048:37005:43;;;;;;;;;2304:6:39;2270:51;;2210;2048:37005:43;;;;;;;;;2210:51:39;;;2150;2048:37005:43;;;;;;;;;2150:51:39;;;2088:53;2048:37005:43;;;;;;;;;2088:53:39;;;2026;2048:37005:43;;;;;;;;;2026:53:39;;;1964;2048:37005:43;;;;;;;;;1964:53:39;;;1900:55;2048:37005:43;;;;;;;;;1900:55:39;;;1851:40;2048:37005:43;;-1:-1:-1;1851:40:39;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;10520:532:44;10615:431;;;;;;;;;;;;;;;;;;10520:532;:::o","linkReferences":{},"immutableReferences":{"40699":[{"start":10609,"length":32},{"start":10981,"length":32}]}},"methodIdentifiers":{"BURN_ACTOR()":"0a6a63f1","EXTRA_DATA_MAX_SIZE()":"029b4646","FIL_USD_PRICE_FEED_ID()":"19c75950","LEAF_SIZE()":"c0e15949","MAX_ENQUEUED_REMOVALS()":"9f8cb3bd","MAX_PIECE_SIZE()":"8a405abc","NO_CHALLENGE_SCHEDULED()":"462dd449","NO_PROVEN_EPOCH()":"f178b1be","PYTH()":"67e406d5","RANDOMNESS_PRECOMPILE()":"15b17570","SECONDS_IN_DAY()":"61a52a36","UPGRADE_INTERFACE_VERSION()":"ad3cb1cc","VERSION()":"ffa1ad74","addPieces(uint256,((bytes),uint256)[],bytes)":"ddea76cc","calculateProofFee(uint256,uint256)":"4903704a","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getActivePieceCount(uint256)":"5353bdfd","getActivePieces(uint256,uint256,uint256)":"39f51544","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getFILUSDPrice()":"4fa27920","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getRandomness(uint256)":"453f4f62","getScheduledRemovals(uint256)":"6fa44692","initialize(uint256)":"fe4b84df","migrate()":"8fd3ab80","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","owner()":"8da5cb5b","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","proxiableUUID()":"52d1902d","renounceOwnership()":"715018a6","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024","transferOwnership(address)":"f2fde38b","upgradeToAndCall(address,bytes)":"4f1ef286"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"target\",\"type\":\"address\"}],\"name\":\"AddressEmptyCode\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"ERC1967InvalidImplementation\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"ERC1967NonPayable\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"FailedCall\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"idx\",\"type\":\"uint256\"},{\"internalType\":\"string\",\"name\":\"msg\",\"type\":\"string\"}],\"name\":\"IndexedError\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"InvalidInitialization\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"NotInitializing\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"OwnableInvalidOwner\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"account\",\"type\":\"address\"}],\"name\":\"OwnableUnauthorizedAccount\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"UUPSUnauthorizedCallContext\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"slot\",\"type\":\"bytes32\"}],\"name\":\"UUPSUnsupportedProxiableUUID\",\"type\":\"error\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"version\",\"type\":\"uint64\"}],\"name\":\"Initialized\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"previousOwner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"OwnershipTransferred\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"reason\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"Upgraded\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"BURN_ACTOR\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"EXTRA_DATA_MAX_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"FIL_USD_PRICE_FEED_ID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"LEAF_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_ENQUEUED_REMOVALS\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_PIECE_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_CHALLENGE_SCHEDULED\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_PROVEN_EPOCH\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PYTH\",\"outputs\":[{\"internalType\":\"contract IPyth\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"RANDOMNESS_PRECOMPILE\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"SECONDS_IN_DAY\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"UPGRADE_INTERFACE_VERSION\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"VERSION\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"piece\",\"type\":\"tuple\"},{\"internalType\":\"uint256\",\"name\":\"rawSize\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceData[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"estimatedGasFee\",\"type\":\"uint256\"}],\"name\":\"calculateProofFee\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getActivePieceCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"activeCount\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"limit\",\"type\":\"uint256\"}],\"name\":\"getActivePieces\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieces\",\"type\":\"tuple[]\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"uint256[]\",\"name\":\"rawSizes\",\"type\":\"uint256[]\"},{\"internalType\":\"bool\",\"name\":\"hasMore\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getFILUSDPrice\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"},{\"internalType\":\"int32\",\"name\":\"\",\"type\":\"int32\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"\",\"type\":\"tuple\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"epoch\",\"type\":\"uint256\"}],\"name\":\"getRandomness\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"_challengeFinality\",\"type\":\"uint256\"}],\"name\":\"initialize\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"migrate\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"owner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"proxiableUUID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"renounceOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"transferOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"name\":\"upgradeToAndCall\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"}],\"devdoc\":{\"errors\":{\"AddressEmptyCode(address)\":[{\"details\":\"There's no code at `target` (it is not a contract).\"}],\"ERC1967InvalidImplementation(address)\":[{\"details\":\"The `implementation` of the proxy is invalid.\"}],\"ERC1967NonPayable()\":[{\"details\":\"An upgrade function sees `msg.value > 0` that may be lost.\"}],\"FailedCall()\":[{\"details\":\"A call to an address target failed. The target may have reverted.\"}],\"InvalidInitialization()\":[{\"details\":\"The contract is already initialized.\"}],\"NotInitializing()\":[{\"details\":\"The contract is not initializing.\"}],\"OwnableInvalidOwner(address)\":[{\"details\":\"The owner is not a valid owner account. (eg. `address(0)`)\"}],\"OwnableUnauthorizedAccount(address)\":[{\"details\":\"The caller account is not authorized to perform an operation.\"}],\"UUPSUnauthorizedCallContext()\":[{\"details\":\"The call is from an unauthorized context.\"}],\"UUPSUnsupportedProxiableUUID(bytes32)\":[{\"details\":\"The storage `slot` is unsupported as a UUID.\"}]},\"events\":{\"Initialized(uint64)\":{\"details\":\"Triggered when the contract has been initialized or reinitialized.\"},\"Upgraded(address)\":{\"details\":\"Emitted when the implementation is upgraded.\"}},\"kind\":\"dev\",\"methods\":{\"constructor\":{\"custom:oz-upgrades-unsafe-allow\":\"constructor\"},\"getActivePieceCount(uint256)\":{\"params\":{\"setId\":\"The data set ID\"},\"returns\":{\"activeCount\":\"The number of active pieces in the data set\"}},\"getActivePieces(uint256,uint256,uint256)\":{\"params\":{\"limit\":\"Maximum number of pieces to return\",\"offset\":\"Starting index for pagination (0-based)\",\"setId\":\"The data set ID\"},\"returns\":{\"hasMore\":\"True if there are more pieces beyond this page\",\"pieceIds\":\"Array of corresponding piece IDs\",\"pieces\":\"Array of active piece CIDs\",\"rawSizes\":\"Array of raw sizes for each piece (in bytes)\"}},\"owner()\":{\"details\":\"Returns the address of the current owner.\"},\"proxiableUUID()\":{\"details\":\"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier.\"},\"renounceOwnership()\":{\"details\":\"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner.\"},\"transferOwnership(address)\":{\"details\":\"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner.\"},\"upgradeToAndCall(address,bytes)\":{\"custom:oz-upgrades-unsafe-allow-reachable\":\"delegatecall\",\"details\":\"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event.\"}},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"getActivePieceCount(uint256)\":{\"notice\":\"Returns the count of active pieces (non-zero leaf count) for a data set\"},\"getActivePieces(uint256,uint256,uint256)\":{\"notice\":\"Returns active pieces (non-zero leaf count) for a data set with pagination\"}},\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/PDPVerifier.sol\":\"PDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol\":{\"keccak256\":\"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6\",\"dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol\":{\"keccak256\":\"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609\",\"dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol\":{\"keccak256\":\"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0\",\"dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol\":{\"keccak256\":\"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9\",\"dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV\"]},\"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol\":{\"keccak256\":\"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d\",\"dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7\"]},\"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol\":{\"keccak256\":\"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196\",\"dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA\"]},\"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol\":{\"keccak256\":\"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a\",\"dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw\"]},\"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol\":{\"keccak256\":\"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa\",\"dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM\"]},\"lib/openzeppelin-contracts/contracts/utils/Address.sol\":{\"keccak256\":\"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23\",\"dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb\"]},\"lib/openzeppelin-contracts/contracts/utils/Errors.sol\":{\"keccak256\":\"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf\",\"dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB\"]},\"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol\":{\"keccak256\":\"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b\",\"dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM\"]},\"lib/pyth-sdk-solidity/IPyth.sol\":{\"keccak256\":\"0x949c65c65fea0578c09a6fc068e09ed1165adede2c835984cefcb25d76de1de2\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://4d7cb071e08e81bb8b113a928f4c2d2b3cdf950ad64c6c7003ea3d874163ca77\",\"dweb:/ipfs/QmRbQchPxRTBMHi7WzLb8XnMGzPDQcWhu7i2u5naUsCRoZ\"]},\"lib/pyth-sdk-solidity/IPythEvents.sol\":{\"keccak256\":\"0x048a35526c2e77d107d43ba336f1dcf31f64cef25ba429ae1f7a0fbc11c23320\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://b75be4c3643b22305995aba71fc92146dbf51fa82d2f9728c515d7749b32dca3\",\"dweb:/ipfs/QmRby4XA9jJQGhxoJ16BTUDuU7BzLFfadbfTgBiQsDgNyZ\"]},\"lib/pyth-sdk-solidity/PythStructs.sol\":{\"keccak256\":\"0x95ff0a6d64517348ef604b8bcf246b561a9445d7e607b8f48491c617cfda9b65\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://fb7f4ffe03be7379d3833c5946e38153de26aef4a4da0323a1ec603787de9eb7\",\"dweb:/ipfs/QmW4WkkLPGjDJrLrW4mYfxtFh8e9KAcPhrnNdxPQsfkS6t\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe9cae265ac70772a6e575e1aee25e046546d16fee65eac956e76aa2c2c4d5d29\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://d48ea8f414e95f53f19f137e5ee95bb327de01ef34be42f8b33a6bc3b88d2535\",\"dweb:/ipfs/QmcMUrKtTW6KpYiiLySBhKakWhUqzifspbzExzRxdoy2A3\"]},\"src/Cids.sol\":{\"keccak256\":\"0xcdfc21c273c5d123e83502248ccd652125ca06465b04a62b12141655090294d7\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://d6663a2a078f0e92fa2555047de4c1fca6eaf47c3448b7d7e53f49d73ed94df9\",\"dweb:/ipfs/QmR6HXrpYqHEKf82WWgVpifRUqygCnSYbSgJjUyt6NVQhg\"]},\"src/Fees.sol\":{\"keccak256\":\"0xbe931ac353310b1f507e30eb4785613169d87146043de6705ba29c9cce39fec2\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://2d09cdebf923161ca3a94a42c56e74a8038cef1b90e933ba48ad916ad89d9ecc\",\"dweb:/ipfs/QmZ6Cx99hAGvuQqkoqAdAuqeT6Fq4Z5Msw7HHGJ8U9it1D\"]},\"src/PDPVerifier.sol\":{\"keccak256\":\"0x6e3ceade2f31e32386e82fedc2faf64746469a7bfafd80b3216274c161fa4879\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://c9f7c353784ef400df67a0771ea5924d50e80e61d28aa65044d8a3d5678baae1\",\"dweb:/ipfs/QmW71sT7wNTLdBUcKknBvuYqsnGYVCmCU7RHydVyrdWSzs\"]},\"src/Proofs.sol\":{\"keccak256\":\"0x9c4a870d9b9d9ea55826fd8b2d2b377ce54f958652189f74a4361949c401069f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://207b6b1587e7ff73068d82b559566b05f52e228efd8fc4e32377d38b4ebd61c2\",\"dweb:/ipfs/QmUjpfXpqsewRVbNTXG4JvF2gVp7Wh17KkcNHbZSRrCdjM\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0x1d127464d67825e324a78214b2219261826a605e7accdbe8beff0fc624612495\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://cc3e1fe67a19aa899779b56a6dad6ed36db01ad8fcdc54667fff14737ebb3ac0\",\"dweb:/ipfs/QmU94UZX7YeaAoUSymuxBEkVg4tFz1S4g5ux7zfWWyVsLy\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x3e784eb8c8eb9fcb860c922e328c4e53d78d3b9bc6a570d0112ee520d2026b16\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://63d33e78ca97ca13878236a5ac6daee67f6fa3a413349a527d77aac3449a659f\",\"dweb:/ipfs/Qme1tSmw4txRwTBG3njFjqdjw2sJjUVnC7Vmwksej9wYKf\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"inputs":[{"internalType":"address","name":"target","type":"address"}],"type":"error","name":"AddressEmptyCode"},{"inputs":[{"internalType":"address","name":"implementation","type":"address"}],"type":"error","name":"ERC1967InvalidImplementation"},{"inputs":[],"type":"error","name":"ERC1967NonPayable"},{"inputs":[],"type":"error","name":"FailedCall"},{"inputs":[{"internalType":"uint256","name":"idx","type":"uint256"},{"internalType":"string","name":"msg","type":"string"}],"type":"error","name":"IndexedError"},{"inputs":[],"type":"error","name":"InvalidInitialization"},{"inputs":[],"type":"error","name":"NotInitializing"},{"inputs":[{"internalType":"address","name":"owner","type":"address"}],"type":"error","name":"OwnableInvalidOwner"},{"inputs":[{"internalType":"address","name":"account","type":"address"}],"type":"error","name":"OwnableUnauthorizedAccount"},{"inputs":[],"type":"error","name":"UUPSUnauthorizedCallContext"},{"inputs":[{"internalType":"bytes32","name":"slot","type":"bytes32"}],"type":"error","name":"UUPSUnsupportedProxiableUUID"},{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"implementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint64","name":"version","type":"uint64","indexed":false}],"type":"event","name":"Initialized","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"address","name":"previousOwner","type":"address","indexed":true},{"internalType":"address","name":"newOwner","type":"address","indexed":true}],"type":"event","name":"OwnershipTransferred","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"reason","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"address","name":"implementation","type":"address","indexed":true}],"type":"event","name":"Upgraded","anonymous":false},{"inputs":[],"stateMutability":"view","type":"function","name":"BURN_ACTOR","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"EXTRA_DATA_MAX_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"FIL_USD_PRICE_FEED_ID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"LEAF_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_ENQUEUED_REMOVALS","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_PIECE_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_CHALLENGE_SCHEDULED","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_PROVEN_EPOCH","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"PYTH","outputs":[{"internalType":"contract IPyth","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"RANDOMNESS_PRECOMPILE","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"SECONDS_IN_DAY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"UPGRADE_INTERFACE_VERSION","outputs":[{"internalType":"string","name":"","type":"string"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"VERSION","outputs":[{"internalType":"string","name":"","type":"string"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.PieceData[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"struct Cids.Cid","name":"piece","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"uint256","name":"rawSize","type":"uint256"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"estimatedGasFee","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"calculateProofFee","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getActivePieceCount","outputs":[{"internalType":"uint256","name":"activeCount","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"},{"internalType":"uint256","name":"limit","type":"uint256"}],"stateMutability":"view","type":"function","name":"getActivePieces","outputs":[{"internalType":"struct Cids.Cid[]","name":"pieces","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"uint256[]","name":"rawSizes","type":"uint256[]"},{"internalType":"bool","name":"hasMore","type":"bool"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"getFILUSDPrice","outputs":[{"internalType":"uint64","name":"","type":"uint64"},{"internalType":"int32","name":"","type":"int32"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"struct Cids.Cid","name":"","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"epoch","type":"uint256"}],"stateMutability":"view","type":"function","name":"getRandomness","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"_challengeFinality","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"initialize"},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"migrate"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[],"stateMutability":"view","type":"function","name":"owner","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[],"stateMutability":"view","type":"function","name":"proxiableUUID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"renounceOwnership"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"},{"inputs":[{"internalType":"address","name":"newOwner","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"transferOwnership"},{"inputs":[{"internalType":"address","name":"newImplementation","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"stateMutability":"payable","type":"function","name":"upgradeToAndCall"}],"devdoc":{"kind":"dev","methods":{"constructor":{"custom:oz-upgrades-unsafe-allow":"constructor"},"getActivePieceCount(uint256)":{"params":{"setId":"The data set ID"},"returns":{"activeCount":"The number of active pieces in the data set"}},"getActivePieces(uint256,uint256,uint256)":{"params":{"limit":"Maximum number of pieces to return","offset":"Starting index for pagination (0-based)","setId":"The data set ID"},"returns":{"hasMore":"True if there are more pieces beyond this page","pieceIds":"Array of corresponding piece IDs","pieces":"Array of active piece CIDs","rawSizes":"Array of raw sizes for each piece (in bytes)"}},"owner()":{"details":"Returns the address of the current owner."},"proxiableUUID()":{"details":"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier."},"renounceOwnership()":{"details":"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner."},"transferOwnership(address)":{"details":"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner."},"upgradeToAndCall(address,bytes)":{"custom:oz-upgrades-unsafe-allow-reachable":"delegatecall","details":"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event."}},"version":1},"userdoc":{"kind":"user","methods":{"getActivePieceCount(uint256)":{"notice":"Returns the count of active pieces (non-zero leaf count) for a data set"},"getActivePieces(uint256,uint256,uint256)":{"notice":"Returns active pieces (non-zero leaf count) for a data set with pagination"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/PDPVerifier.sol":"PDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol":{"keccak256":"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a","urls":["bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6","dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol":{"keccak256":"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b","urls":["bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609","dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol":{"keccak256":"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a","urls":["bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0","dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol":{"keccak256":"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397","urls":["bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9","dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol":{"keccak256":"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486","urls":["bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d","dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol":{"keccak256":"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d","urls":["bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196","dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol":{"keccak256":"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128","urls":["bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a","dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol":{"keccak256":"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c","urls":["bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa","dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Address.sol":{"keccak256":"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5","urls":["bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23","dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Errors.sol":{"keccak256":"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123","urls":["bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf","dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol":{"keccak256":"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97","urls":["bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b","dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM"],"license":"MIT"},"lib/pyth-sdk-solidity/IPyth.sol":{"keccak256":"0x949c65c65fea0578c09a6fc068e09ed1165adede2c835984cefcb25d76de1de2","urls":["bzz-raw://4d7cb071e08e81bb8b113a928f4c2d2b3cdf950ad64c6c7003ea3d874163ca77","dweb:/ipfs/QmRbQchPxRTBMHi7WzLb8XnMGzPDQcWhu7i2u5naUsCRoZ"],"license":"Apache-2.0"},"lib/pyth-sdk-solidity/IPythEvents.sol":{"keccak256":"0x048a35526c2e77d107d43ba336f1dcf31f64cef25ba429ae1f7a0fbc11c23320","urls":["bzz-raw://b75be4c3643b22305995aba71fc92146dbf51fa82d2f9728c515d7749b32dca3","dweb:/ipfs/QmRby4XA9jJQGhxoJ16BTUDuU7BzLFfadbfTgBiQsDgNyZ"],"license":"Apache-2.0"},"lib/pyth-sdk-solidity/PythStructs.sol":{"keccak256":"0x95ff0a6d64517348ef604b8bcf246b561a9445d7e607b8f48491c617cfda9b65","urls":["bzz-raw://fb7f4ffe03be7379d3833c5946e38153de26aef4a4da0323a1ec603787de9eb7","dweb:/ipfs/QmW4WkkLPGjDJrLrW4mYfxtFh8e9KAcPhrnNdxPQsfkS6t"],"license":"Apache-2.0"},"src/BitOps.sol":{"keccak256":"0xe9cae265ac70772a6e575e1aee25e046546d16fee65eac956e76aa2c2c4d5d29","urls":["bzz-raw://d48ea8f414e95f53f19f137e5ee95bb327de01ef34be42f8b33a6bc3b88d2535","dweb:/ipfs/QmcMUrKtTW6KpYiiLySBhKakWhUqzifspbzExzRxdoy2A3"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0xcdfc21c273c5d123e83502248ccd652125ca06465b04a62b12141655090294d7","urls":["bzz-raw://d6663a2a078f0e92fa2555047de4c1fca6eaf47c3448b7d7e53f49d73ed94df9","dweb:/ipfs/QmR6HXrpYqHEKf82WWgVpifRUqygCnSYbSgJjUyt6NVQhg"],"license":"Apache-2.0 OR MIT"},"src/Fees.sol":{"keccak256":"0xbe931ac353310b1f507e30eb4785613169d87146043de6705ba29c9cce39fec2","urls":["bzz-raw://2d09cdebf923161ca3a94a42c56e74a8038cef1b90e933ba48ad916ad89d9ecc","dweb:/ipfs/QmZ6Cx99hAGvuQqkoqAdAuqeT6Fq4Z5Msw7HHGJ8U9it1D"],"license":"Apache-2.0 OR MIT"},"src/PDPVerifier.sol":{"keccak256":"0x6e3ceade2f31e32386e82fedc2faf64746469a7bfafd80b3216274c161fa4879","urls":["bzz-raw://c9f7c353784ef400df67a0771ea5924d50e80e61d28aa65044d8a3d5678baae1","dweb:/ipfs/QmW71sT7wNTLdBUcKknBvuYqsnGYVCmCU7RHydVyrdWSzs"],"license":"Apache-2.0 OR MIT"},"src/Proofs.sol":{"keccak256":"0x9c4a870d9b9d9ea55826fd8b2d2b377ce54f958652189f74a4361949c401069f","urls":["bzz-raw://207b6b1587e7ff73068d82b559566b05f52e228efd8fc4e32377d38b4ebd61c2","dweb:/ipfs/QmUjpfXpqsewRVbNTXG4JvF2gVp7Wh17KkcNHbZSRrCdjM"],"license":"MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0x1d127464d67825e324a78214b2219261826a605e7accdbe8beff0fc624612495","urls":["bzz-raw://cc3e1fe67a19aa899779b56a6dad6ed36db01ad8fcdc54667fff14737ebb3ac0","dweb:/ipfs/QmU94UZX7YeaAoUSymuxBEkVg4tFz1S4g5ux7zfWWyVsLy"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x3e784eb8c8eb9fcb860c922e328c4e53d78d3b9bc6a570d0112ee520d2026b16","urls":["bzz-raw://63d33e78ca97ca13878236a5ac6daee67f6fa3a413349a527d77aac3449a659f","dweb:/ipfs/Qme1tSmw4txRwTBG3njFjqdjw2sJjUVnC7Vmwksej9wYKf"],"license":"MIT"}},"version":1},"id":43} \ No newline at end of file diff --git a/pdp/contract/README.md b/pdp/contract/README.md new file mode 100644 index 000000000..489519b2d --- /dev/null +++ b/pdp/contract/README.md @@ -0,0 +1,110 @@ +# Guide for Generating Go Bindings Using `abigen` + +This guide explains how to use the `abigen` tool to generate Go bindings for Ethereum smart contracts. These bindings allow you to interact with contracts in Go programs. The smart contract ABIs (Application Binary Interfaces) are retrieved from the source repository and updated after being processed with `make build`. + +--- + +## Prerequisites + +1. **Install `abigen`:** + Install `abigen` from the Go Ethereum (geth) toolset. You can install it via the following command: + + ```bash + go install github.com/ethereum/go-ethereum/cmd/abigen@latest + ``` + +2. **Ensure Forge (`foundry`) is Installed:** + The `make build` step requires the Forge tool (from Foundry). Install it via: + + ```bash + curl -L https://foundry.paradigm.xyz | bash + foundryup + ``` + +3. **Clone the Repository:** + Clone the repository where the smart contract code resides: + + ```bash + git clone https://github.com/FilOzone/pdp.git + cd pdp + ``` + +--- + +## Steps to Generate Go Bindings + +### Step 1: Build the Contracts using `make build` + +In the root of the cloned repository, run: + +```bash +make build +``` + +This command will create the `out/` directory containing the compiled contract artifacts, such as `IPDPProvingSchedule.json` and `PDPVerifier.json`. + +--- + +### Step 2: Extract ABIs from Compiled Artifacts + +Navigate to the `out/` directory and extract the ABI from the compiled JSON files for the required contracts. Use the `jq` tool: + +#### For `IPDPProvingSchedule` ABI: + +Run: + +```bash +jq '.abi' out/IPDPProvingSchedule.sol/IPDPProvingSchedule.json > pdp/contract/IPDPProvingSchedule.abi +``` + +#### For `PDPVerifier` ABI: + +Run: + +```bash +jq '.abi' out/PDPVerifier.sol/PDPVerifier.json > pdp/contract/PDPVerifier.abi +``` + +Ensure that the respective `.abi` files are updated in the `pdp/contract/` directory. + +--- + +### Step 3: Generate Go Bindings Using `abigen` + +Use the `abigen` command-line tool to generate the Go bindings for the parsed ABIs. + +#### For `IPDPProvingSchedule` Contract: + +Run: + +```bash +abigen --abi pdp/contract/IPDPProvingSchedule.abi --pkg contract --type IPDPProvingSchedule --out pdp/contract/pdp_proving_schedule.go +``` + +- `--abi`: Path to the `.abi` file for the contract. +- `--pkg`: Package name in the generated Go code (use the relevant package name, e.g., `contract` in this case). +- `--type`: The Go struct type for this contract (use descriptive names like `IPDPProvingSchedule`). +- `--out`: Output file path for the generated Go file (e.g., `pdp_proving_schedule.go`). + +--- + +#### For `PDPVerifier` Contract: + +Run: + +```bash +abigen --abi pdp/contract/PDPVerifier.abi --pkg contract --type PDPVerifier --out pdp/contract/pdp_verifier.go +``` + +--- + +### Step 4: Verify the Outputs + +After running the `abigen` commands, the Go files (`pdp_proving_schedule.go` and `pdp_verifier.go`) will be generated in the `pdp/contract/` directory. These files contain the Go bindings that can be used in Go applications to interact with the corresponding smart contracts. + +--- + +### Notes + +- **ABI Files:** Ensure that the `.abi` files are correct and up to date by extracting them directly from the compiled JSON artifacts. +- **Code Organization:** Keep both the generated Go files and ABI files in a structured directory layout for easier maintenance (e.g., under `pdp/contract/`). \ No newline at end of file diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index b62c716f8..d718116f0 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -12,7 +12,7 @@ import ( ) const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" -const PDPCalibnet = "0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC" +const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" const PDPTestNet = "Change Me" type PDPContracts struct { diff --git a/pdp/contract/pdp_verifier.go b/pdp/contract/pdp_verifier.go index 64855170e..9a9de2a37 100644 --- a/pdp/contract/pdp_verifier.go +++ b/pdp/contract/pdp_verifier.go @@ -34,27 +34,27 @@ type CidsCid struct { Data []byte } -// PDPVerifierProof is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierProof struct { - Leaf [32]byte - Proof [][32]byte +// PDPVerifierPieceData is an auto generated low-level Go binding around an user-defined struct. +type PDPVerifierPieceData struct { + Piece CidsCid + RawSize *big.Int } -// PDPVerifierRootData is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierRootData struct { - Root CidsCid - RawSize *big.Int +// PDPVerifierPieceIdAndOffset is an auto generated low-level Go binding around an user-defined struct. +type PDPVerifierPieceIdAndOffset struct { + PieceId *big.Int + Offset *big.Int } -// PDPVerifierRootIdAndOffset is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierRootIdAndOffset struct { - RootId *big.Int - Offset *big.Int +// PDPVerifierProof is an auto generated low-level Go binding around an user-defined struct. +type PDPVerifierProof struct { + Leaf [32]byte + Proof [][32]byte } // PDPVerifierMetaData contains all meta data concerning the PDPVerifier contract. var PDPVerifierMetaData = &bind.MetaData{ - ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ROOT_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addRoots\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootData\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.RootData[]\",\"components\":[{\"name\":\"root\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"rawSize\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"claimProofSetOwnership\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createProofSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"deleteProofSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findRootIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.RootIdAndOffset[]\",\"components\":[{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextProofSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextRootId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getProofSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getProofSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getProofSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getProofSetOwner\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRootCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRootLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proofSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeProofSetOwner\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"rootChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"rootLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"scheduleRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"rootIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"Debug\",\"inputs\":[{\"name\":\"message\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"value\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structPDPVerifier.RootIdAndOffset[]\",\"components\":[{\"name\":\"rootId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"owner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofSetOwnerChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"RootsAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"rootIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"RootsRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"rootIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", + ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.PieceData[]\",\"components\":[{\"name\":\"piece\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"rawSize\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"event\",\"name\":\"Debug\",\"inputs\":[{\"name\":\"message\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"value\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structPDPVerifier.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", } // PDPVerifierABI is the input ABI used to generate the binding from. @@ -358,12 +358,12 @@ func (_PDPVerifier *PDPVerifierCallerSession) MAXENQUEUEDREMOVALS() (*big.Int, e return _PDPVerifier.Contract.MAXENQUEUEDREMOVALS(&_PDPVerifier.CallOpts) } -// MAXROOTSIZE is a free data retrieval call binding the contract method 0x16e2bcd5. +// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. // -// Solidity: function MAX_ROOT_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) MAXROOTSIZE(opts *bind.CallOpts) (*big.Int, error) { +// Solidity: function MAX_PIECE_SIZE() view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) MAXPIECESIZE(opts *bind.CallOpts) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "MAX_ROOT_SIZE") + err := _PDPVerifier.contract.Call(opts, &out, "MAX_PIECE_SIZE") if err != nil { return *new(*big.Int), err @@ -375,18 +375,18 @@ func (_PDPVerifier *PDPVerifierCaller) MAXROOTSIZE(opts *bind.CallOpts) (*big.In } -// MAXROOTSIZE is a free data retrieval call binding the contract method 0x16e2bcd5. +// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. // -// Solidity: function MAX_ROOT_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) MAXROOTSIZE() (*big.Int, error) { - return _PDPVerifier.Contract.MAXROOTSIZE(&_PDPVerifier.CallOpts) +// Solidity: function MAX_PIECE_SIZE() view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) MAXPIECESIZE() (*big.Int, error) { + return _PDPVerifier.Contract.MAXPIECESIZE(&_PDPVerifier.CallOpts) } -// MAXROOTSIZE is a free data retrieval call binding the contract method 0x16e2bcd5. +// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. // -// Solidity: function MAX_ROOT_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) MAXROOTSIZE() (*big.Int, error) { - return _PDPVerifier.Contract.MAXROOTSIZE(&_PDPVerifier.CallOpts) +// Solidity: function MAX_PIECE_SIZE() view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) MAXPIECESIZE() (*big.Int, error) { + return _PDPVerifier.Contract.MAXPIECESIZE(&_PDPVerifier.CallOpts) } // NOCHALLENGESCHEDULED is a free data retrieval call binding the contract method 0x462dd449. @@ -606,35 +606,152 @@ func (_PDPVerifier *PDPVerifierCallerSession) CalculateProofFee(setId *big.Int, return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.CallOpts, setId, estimatedGasFee) } -// FindRootIds is a free data retrieval call binding the contract method 0x0528a55b. +// DataSetLive is a free data retrieval call binding the contract method 0xca759f27. +// +// Solidity: function dataSetLive(uint256 setId) view returns(bool) +func (_PDPVerifier *PDPVerifierCaller) DataSetLive(opts *bind.CallOpts, setId *big.Int) (bool, error) { + var out []interface{} + err := _PDPVerifier.contract.Call(opts, &out, "dataSetLive", setId) + + if err != nil { + return *new(bool), err + } + + out0 := *abi.ConvertType(out[0], new(bool)).(*bool) + + return out0, err + +} + +// DataSetLive is a free data retrieval call binding the contract method 0xca759f27. +// +// Solidity: function dataSetLive(uint256 setId) view returns(bool) +func (_PDPVerifier *PDPVerifierSession) DataSetLive(setId *big.Int) (bool, error) { + return _PDPVerifier.Contract.DataSetLive(&_PDPVerifier.CallOpts, setId) +} + +// DataSetLive is a free data retrieval call binding the contract method 0xca759f27. +// +// Solidity: function dataSetLive(uint256 setId) view returns(bool) +func (_PDPVerifier *PDPVerifierCallerSession) DataSetLive(setId *big.Int) (bool, error) { + return _PDPVerifier.Contract.DataSetLive(&_PDPVerifier.CallOpts, setId) +} + +// FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. +// +// Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) +func (_PDPVerifier *PDPVerifierCaller) FindPieceIds(opts *bind.CallOpts, setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { + var out []interface{} + err := _PDPVerifier.contract.Call(opts, &out, "findPieceIds", setId, leafIndexs) + + if err != nil { + return *new([]PDPVerifierPieceIdAndOffset), err + } + + out0 := *abi.ConvertType(out[0], new([]PDPVerifierPieceIdAndOffset)).(*[]PDPVerifierPieceIdAndOffset) + + return out0, err + +} + +// FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. // -// Solidity: function findRootIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierCaller) FindRootIds(opts *bind.CallOpts, setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierRootIdAndOffset, error) { +// Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) +func (_PDPVerifier *PDPVerifierSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { + return _PDPVerifier.Contract.FindPieceIds(&_PDPVerifier.CallOpts, setId, leafIndexs) +} + +// FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. +// +// Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) +func (_PDPVerifier *PDPVerifierCallerSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { + return _PDPVerifier.Contract.FindPieceIds(&_PDPVerifier.CallOpts, setId, leafIndexs) +} + +// GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. +// +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetActivePieceCount(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "findRootIds", setId, leafIndexs) + err := _PDPVerifier.contract.Call(opts, &out, "getActivePieceCount", setId) if err != nil { - return *new([]PDPVerifierRootIdAndOffset), err + return *new(*big.Int), err } - out0 := *abi.ConvertType(out[0], new([]PDPVerifierRootIdAndOffset)).(*[]PDPVerifierRootIdAndOffset) + out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) return out0, err } -// FindRootIds is a free data retrieval call binding the contract method 0x0528a55b. +// GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. +// +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetActivePieceCount(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetActivePieceCount(&_PDPVerifier.CallOpts, setId) +} + +// GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. +// +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetActivePieceCount(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetActivePieceCount(&_PDPVerifier.CallOpts, setId) +} + +// GetActivePieces is a free data retrieval call binding the contract method 0x39f51544. +// +// Solidity: function getActivePieces(uint256 setId, uint256 offset, uint256 limit) view returns((bytes)[] pieces, uint256[] pieceIds, uint256[] rawSizes, bool hasMore) +func (_PDPVerifier *PDPVerifierCaller) GetActivePieces(opts *bind.CallOpts, setId *big.Int, offset *big.Int, limit *big.Int) (struct { + Pieces []CidsCid + PieceIds []*big.Int + RawSizes []*big.Int + HasMore bool +}, error) { + var out []interface{} + err := _PDPVerifier.contract.Call(opts, &out, "getActivePieces", setId, offset, limit) + + outstruct := new(struct { + Pieces []CidsCid + PieceIds []*big.Int + RawSizes []*big.Int + HasMore bool + }) + if err != nil { + return *outstruct, err + } + + outstruct.Pieces = *abi.ConvertType(out[0], new([]CidsCid)).(*[]CidsCid) + outstruct.PieceIds = *abi.ConvertType(out[1], new([]*big.Int)).(*[]*big.Int) + outstruct.RawSizes = *abi.ConvertType(out[2], new([]*big.Int)).(*[]*big.Int) + outstruct.HasMore = *abi.ConvertType(out[3], new(bool)).(*bool) + + return *outstruct, err + +} + +// GetActivePieces is a free data retrieval call binding the contract method 0x39f51544. // -// Solidity: function findRootIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierSession) FindRootIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierRootIdAndOffset, error) { - return _PDPVerifier.Contract.FindRootIds(&_PDPVerifier.CallOpts, setId, leafIndexs) +// Solidity: function getActivePieces(uint256 setId, uint256 offset, uint256 limit) view returns((bytes)[] pieces, uint256[] pieceIds, uint256[] rawSizes, bool hasMore) +func (_PDPVerifier *PDPVerifierSession) GetActivePieces(setId *big.Int, offset *big.Int, limit *big.Int) (struct { + Pieces []CidsCid + PieceIds []*big.Int + RawSizes []*big.Int + HasMore bool +}, error) { + return _PDPVerifier.Contract.GetActivePieces(&_PDPVerifier.CallOpts, setId, offset, limit) } -// FindRootIds is a free data retrieval call binding the contract method 0x0528a55b. +// GetActivePieces is a free data retrieval call binding the contract method 0x39f51544. // -// Solidity: function findRootIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierCallerSession) FindRootIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierRootIdAndOffset, error) { - return _PDPVerifier.Contract.FindRootIds(&_PDPVerifier.CallOpts, setId, leafIndexs) +// Solidity: function getActivePieces(uint256 setId, uint256 offset, uint256 limit) view returns((bytes)[] pieces, uint256[] pieceIds, uint256[] rawSizes, bool hasMore) +func (_PDPVerifier *PDPVerifierCallerSession) GetActivePieces(setId *big.Int, offset *big.Int, limit *big.Int) (struct { + Pieces []CidsCid + PieceIds []*big.Int + RawSizes []*big.Int + HasMore bool +}, error) { + return _PDPVerifier.Contract.GetActivePieces(&_PDPVerifier.CallOpts, setId, offset, limit) } // GetChallengeFinality is a free data retrieval call binding the contract method 0xf83758fe. @@ -699,44 +816,43 @@ func (_PDPVerifier *PDPVerifierCallerSession) GetChallengeRange(setId *big.Int) return _PDPVerifier.Contract.GetChallengeRange(&_PDPVerifier.CallOpts, setId) } -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// GetDataSetLastProvenEpoch is a free data retrieval call binding the contract method 0x04595c1a. // -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierCaller) GetFILUSDPrice(opts *bind.CallOpts) (uint64, int32, error) { +// Solidity: function getDataSetLastProvenEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetDataSetLastProvenEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getFILUSDPrice") + err := _PDPVerifier.contract.Call(opts, &out, "getDataSetLastProvenEpoch", setId) if err != nil { - return *new(uint64), *new(int32), err + return *new(*big.Int), err } - out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) - out1 := *abi.ConvertType(out[1], new(int32)).(*int32) + out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) - return out0, out1, err + return out0, err } -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// GetDataSetLastProvenEpoch is a free data retrieval call binding the contract method 0x04595c1a. // -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (uint64, int32, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) +// Solidity: function getDataSetLastProvenEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetDataSetLastProvenEpoch(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetDataSetLastProvenEpoch(&_PDPVerifier.CallOpts, setId) } -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// GetDataSetLastProvenEpoch is a free data retrieval call binding the contract method 0x04595c1a. // -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierCallerSession) GetFILUSDPrice() (uint64, int32, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) +// Solidity: function getDataSetLastProvenEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetLastProvenEpoch(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetDataSetLastProvenEpoch(&_PDPVerifier.CallOpts, setId) } -// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. +// GetDataSetLeafCount is a free data retrieval call binding the contract method 0xa531998c. // -// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetNextChallengeEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { +// Solidity: function getDataSetLeafCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetDataSetLeafCount(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getNextChallengeEpoch", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getDataSetLeafCount", setId) if err != nil { return *new(*big.Int), err @@ -748,119 +864,121 @@ func (_PDPVerifier *PDPVerifierCaller) GetNextChallengeEpoch(opts *bind.CallOpts } -// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. +// GetDataSetLeafCount is a free data retrieval call binding the contract method 0xa531998c. // -// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetNextChallengeEpoch(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetNextChallengeEpoch(&_PDPVerifier.CallOpts, setId) +// Solidity: function getDataSetLeafCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetDataSetLeafCount(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetDataSetLeafCount(&_PDPVerifier.CallOpts, setId) } -// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. +// GetDataSetLeafCount is a free data retrieval call binding the contract method 0xa531998c. // -// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetNextChallengeEpoch(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetNextChallengeEpoch(&_PDPVerifier.CallOpts, setId) +// Solidity: function getDataSetLeafCount(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetLeafCount(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetDataSetLeafCount(&_PDPVerifier.CallOpts, setId) } -// GetNextProofSetId is a free data retrieval call binding the contract method 0x8ea417e5. +// GetDataSetListener is a free data retrieval call binding the contract method 0x2b3129bb. // -// Solidity: function getNextProofSetId() view returns(uint64) -func (_PDPVerifier *PDPVerifierCaller) GetNextProofSetId(opts *bind.CallOpts) (uint64, error) { +// Solidity: function getDataSetListener(uint256 setId) view returns(address) +func (_PDPVerifier *PDPVerifierCaller) GetDataSetListener(opts *bind.CallOpts, setId *big.Int) (common.Address, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getNextProofSetId") + err := _PDPVerifier.contract.Call(opts, &out, "getDataSetListener", setId) if err != nil { - return *new(uint64), err + return *new(common.Address), err } - out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) + out0 := *abi.ConvertType(out[0], new(common.Address)).(*common.Address) return out0, err } -// GetNextProofSetId is a free data retrieval call binding the contract method 0x8ea417e5. +// GetDataSetListener is a free data retrieval call binding the contract method 0x2b3129bb. // -// Solidity: function getNextProofSetId() view returns(uint64) -func (_PDPVerifier *PDPVerifierSession) GetNextProofSetId() (uint64, error) { - return _PDPVerifier.Contract.GetNextProofSetId(&_PDPVerifier.CallOpts) +// Solidity: function getDataSetListener(uint256 setId) view returns(address) +func (_PDPVerifier *PDPVerifierSession) GetDataSetListener(setId *big.Int) (common.Address, error) { + return _PDPVerifier.Contract.GetDataSetListener(&_PDPVerifier.CallOpts, setId) } -// GetNextProofSetId is a free data retrieval call binding the contract method 0x8ea417e5. +// GetDataSetListener is a free data retrieval call binding the contract method 0x2b3129bb. // -// Solidity: function getNextProofSetId() view returns(uint64) -func (_PDPVerifier *PDPVerifierCallerSession) GetNextProofSetId() (uint64, error) { - return _PDPVerifier.Contract.GetNextProofSetId(&_PDPVerifier.CallOpts) +// Solidity: function getDataSetListener(uint256 setId) view returns(address) +func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetListener(setId *big.Int) (common.Address, error) { + return _PDPVerifier.Contract.GetDataSetListener(&_PDPVerifier.CallOpts, setId) } -// GetNextRootId is a free data retrieval call binding the contract method 0xd49245c1. +// GetDataSetStorageProvider is a free data retrieval call binding the contract method 0x21b7cd1c. // -// Solidity: function getNextRootId(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetNextRootId(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { +// Solidity: function getDataSetStorageProvider(uint256 setId) view returns(address, address) +func (_PDPVerifier *PDPVerifierCaller) GetDataSetStorageProvider(opts *bind.CallOpts, setId *big.Int) (common.Address, common.Address, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getNextRootId", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getDataSetStorageProvider", setId) if err != nil { - return *new(*big.Int), err + return *new(common.Address), *new(common.Address), err } - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) + out0 := *abi.ConvertType(out[0], new(common.Address)).(*common.Address) + out1 := *abi.ConvertType(out[1], new(common.Address)).(*common.Address) - return out0, err + return out0, out1, err } -// GetNextRootId is a free data retrieval call binding the contract method 0xd49245c1. +// GetDataSetStorageProvider is a free data retrieval call binding the contract method 0x21b7cd1c. // -// Solidity: function getNextRootId(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetNextRootId(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetNextRootId(&_PDPVerifier.CallOpts, setId) +// Solidity: function getDataSetStorageProvider(uint256 setId) view returns(address, address) +func (_PDPVerifier *PDPVerifierSession) GetDataSetStorageProvider(setId *big.Int) (common.Address, common.Address, error) { + return _PDPVerifier.Contract.GetDataSetStorageProvider(&_PDPVerifier.CallOpts, setId) } -// GetNextRootId is a free data retrieval call binding the contract method 0xd49245c1. +// GetDataSetStorageProvider is a free data retrieval call binding the contract method 0x21b7cd1c. // -// Solidity: function getNextRootId(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetNextRootId(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetNextRootId(&_PDPVerifier.CallOpts, setId) +// Solidity: function getDataSetStorageProvider(uint256 setId) view returns(address, address) +func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetStorageProvider(setId *big.Int) (common.Address, common.Address, error) { + return _PDPVerifier.Contract.GetDataSetStorageProvider(&_PDPVerifier.CallOpts, setId) } -// GetProofSetLastProvenEpoch is a free data retrieval call binding the contract method 0xfaa67163. +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. // -// Solidity: function getProofSetLastProvenEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetProofSetLastProvenEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierCaller) GetFILUSDPrice(opts *bind.CallOpts) (uint64, int32, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getProofSetLastProvenEpoch", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getFILUSDPrice") if err != nil { - return *new(*big.Int), err + return *new(uint64), *new(int32), err } - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) + out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) + out1 := *abi.ConvertType(out[1], new(int32)).(*int32) - return out0, err + return out0, out1, err } -// GetProofSetLastProvenEpoch is a free data retrieval call binding the contract method 0xfaa67163. +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. // -// Solidity: function getProofSetLastProvenEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetProofSetLastProvenEpoch(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetProofSetLastProvenEpoch(&_PDPVerifier.CallOpts, setId) +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (uint64, int32, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) } -// GetProofSetLastProvenEpoch is a free data retrieval call binding the contract method 0xfaa67163. +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. // -// Solidity: function getProofSetLastProvenEpoch(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetProofSetLastProvenEpoch(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetProofSetLastProvenEpoch(&_PDPVerifier.CallOpts, setId) +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierCallerSession) GetFILUSDPrice() (uint64, int32, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) } -// GetProofSetLeafCount is a free data retrieval call binding the contract method 0x3f84135f. +// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. // -// Solidity: function getProofSetLeafCount(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetProofSetLeafCount(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { +// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetNextChallengeEpoch(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getProofSetLeafCount", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getNextChallengeEpoch", setId) if err != nil { return *new(*big.Int), err @@ -872,151 +990,150 @@ func (_PDPVerifier *PDPVerifierCaller) GetProofSetLeafCount(opts *bind.CallOpts, } -// GetProofSetLeafCount is a free data retrieval call binding the contract method 0x3f84135f. +// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. // -// Solidity: function getProofSetLeafCount(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetProofSetLeafCount(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetProofSetLeafCount(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetNextChallengeEpoch(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetNextChallengeEpoch(&_PDPVerifier.CallOpts, setId) } -// GetProofSetLeafCount is a free data retrieval call binding the contract method 0x3f84135f. +// GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. // -// Solidity: function getProofSetLeafCount(uint256 setId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetProofSetLeafCount(setId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetProofSetLeafCount(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetNextChallengeEpoch(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetNextChallengeEpoch(&_PDPVerifier.CallOpts, setId) } -// GetProofSetListener is a free data retrieval call binding the contract method 0x31601226. +// GetNextDataSetId is a free data retrieval call binding the contract method 0x442cded3. // -// Solidity: function getProofSetListener(uint256 setId) view returns(address) -func (_PDPVerifier *PDPVerifierCaller) GetProofSetListener(opts *bind.CallOpts, setId *big.Int) (common.Address, error) { +// Solidity: function getNextDataSetId() view returns(uint64) +func (_PDPVerifier *PDPVerifierCaller) GetNextDataSetId(opts *bind.CallOpts) (uint64, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getProofSetListener", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getNextDataSetId") if err != nil { - return *new(common.Address), err + return *new(uint64), err } - out0 := *abi.ConvertType(out[0], new(common.Address)).(*common.Address) + out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) return out0, err } -// GetProofSetListener is a free data retrieval call binding the contract method 0x31601226. +// GetNextDataSetId is a free data retrieval call binding the contract method 0x442cded3. // -// Solidity: function getProofSetListener(uint256 setId) view returns(address) -func (_PDPVerifier *PDPVerifierSession) GetProofSetListener(setId *big.Int) (common.Address, error) { - return _PDPVerifier.Contract.GetProofSetListener(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextDataSetId() view returns(uint64) +func (_PDPVerifier *PDPVerifierSession) GetNextDataSetId() (uint64, error) { + return _PDPVerifier.Contract.GetNextDataSetId(&_PDPVerifier.CallOpts) } -// GetProofSetListener is a free data retrieval call binding the contract method 0x31601226. +// GetNextDataSetId is a free data retrieval call binding the contract method 0x442cded3. // -// Solidity: function getProofSetListener(uint256 setId) view returns(address) -func (_PDPVerifier *PDPVerifierCallerSession) GetProofSetListener(setId *big.Int) (common.Address, error) { - return _PDPVerifier.Contract.GetProofSetListener(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextDataSetId() view returns(uint64) +func (_PDPVerifier *PDPVerifierCallerSession) GetNextDataSetId() (uint64, error) { + return _PDPVerifier.Contract.GetNextDataSetId(&_PDPVerifier.CallOpts) } -// GetProofSetOwner is a free data retrieval call binding the contract method 0x4726075b. +// GetNextPieceId is a free data retrieval call binding the contract method 0x1c5ae80f. // -// Solidity: function getProofSetOwner(uint256 setId) view returns(address, address) -func (_PDPVerifier *PDPVerifierCaller) GetProofSetOwner(opts *bind.CallOpts, setId *big.Int) (common.Address, common.Address, error) { +// Solidity: function getNextPieceId(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetNextPieceId(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getProofSetOwner", setId) + err := _PDPVerifier.contract.Call(opts, &out, "getNextPieceId", setId) if err != nil { - return *new(common.Address), *new(common.Address), err + return *new(*big.Int), err } - out0 := *abi.ConvertType(out[0], new(common.Address)).(*common.Address) - out1 := *abi.ConvertType(out[1], new(common.Address)).(*common.Address) + out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) - return out0, out1, err + return out0, err } -// GetProofSetOwner is a free data retrieval call binding the contract method 0x4726075b. +// GetNextPieceId is a free data retrieval call binding the contract method 0x1c5ae80f. // -// Solidity: function getProofSetOwner(uint256 setId) view returns(address, address) -func (_PDPVerifier *PDPVerifierSession) GetProofSetOwner(setId *big.Int) (common.Address, common.Address, error) { - return _PDPVerifier.Contract.GetProofSetOwner(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextPieceId(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetNextPieceId(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetNextPieceId(&_PDPVerifier.CallOpts, setId) } -// GetProofSetOwner is a free data retrieval call binding the contract method 0x4726075b. +// GetNextPieceId is a free data retrieval call binding the contract method 0x1c5ae80f. // -// Solidity: function getProofSetOwner(uint256 setId) view returns(address, address) -func (_PDPVerifier *PDPVerifierCallerSession) GetProofSetOwner(setId *big.Int) (common.Address, common.Address, error) { - return _PDPVerifier.Contract.GetProofSetOwner(&_PDPVerifier.CallOpts, setId) +// Solidity: function getNextPieceId(uint256 setId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetNextPieceId(setId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetNextPieceId(&_PDPVerifier.CallOpts, setId) } -// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. +// GetPieceCid is a free data retrieval call binding the contract method 0x25bbbedf. // -// Solidity: function getRandomness(uint256 epoch) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetRandomness(opts *bind.CallOpts, epoch *big.Int) (*big.Int, error) { +// Solidity: function getPieceCid(uint256 setId, uint256 pieceId) view returns((bytes)) +func (_PDPVerifier *PDPVerifierCaller) GetPieceCid(opts *bind.CallOpts, setId *big.Int, pieceId *big.Int) (CidsCid, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getRandomness", epoch) + err := _PDPVerifier.contract.Call(opts, &out, "getPieceCid", setId, pieceId) if err != nil { - return *new(*big.Int), err + return *new(CidsCid), err } - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) + out0 := *abi.ConvertType(out[0], new(CidsCid)).(*CidsCid) return out0, err } -// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. +// GetPieceCid is a free data retrieval call binding the contract method 0x25bbbedf. // -// Solidity: function getRandomness(uint256 epoch) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetRandomness(epoch *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetRandomness(&_PDPVerifier.CallOpts, epoch) +// Solidity: function getPieceCid(uint256 setId, uint256 pieceId) view returns((bytes)) +func (_PDPVerifier *PDPVerifierSession) GetPieceCid(setId *big.Int, pieceId *big.Int) (CidsCid, error) { + return _PDPVerifier.Contract.GetPieceCid(&_PDPVerifier.CallOpts, setId, pieceId) } -// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. +// GetPieceCid is a free data retrieval call binding the contract method 0x25bbbedf. // -// Solidity: function getRandomness(uint256 epoch) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetRandomness(epoch *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetRandomness(&_PDPVerifier.CallOpts, epoch) +// Solidity: function getPieceCid(uint256 setId, uint256 pieceId) view returns((bytes)) +func (_PDPVerifier *PDPVerifierCallerSession) GetPieceCid(setId *big.Int, pieceId *big.Int) (CidsCid, error) { + return _PDPVerifier.Contract.GetPieceCid(&_PDPVerifier.CallOpts, setId, pieceId) } -// GetRootCid is a free data retrieval call binding the contract method 0x3b7ae913. +// GetPieceLeafCount is a free data retrieval call binding the contract method 0x0cd7b880. // -// Solidity: function getRootCid(uint256 setId, uint256 rootId) view returns((bytes)) -func (_PDPVerifier *PDPVerifierCaller) GetRootCid(opts *bind.CallOpts, setId *big.Int, rootId *big.Int) (CidsCid, error) { +// Solidity: function getPieceLeafCount(uint256 setId, uint256 pieceId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetPieceLeafCount(opts *bind.CallOpts, setId *big.Int, pieceId *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getRootCid", setId, rootId) + err := _PDPVerifier.contract.Call(opts, &out, "getPieceLeafCount", setId, pieceId) if err != nil { - return *new(CidsCid), err + return *new(*big.Int), err } - out0 := *abi.ConvertType(out[0], new(CidsCid)).(*CidsCid) + out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) return out0, err } -// GetRootCid is a free data retrieval call binding the contract method 0x3b7ae913. +// GetPieceLeafCount is a free data retrieval call binding the contract method 0x0cd7b880. // -// Solidity: function getRootCid(uint256 setId, uint256 rootId) view returns((bytes)) -func (_PDPVerifier *PDPVerifierSession) GetRootCid(setId *big.Int, rootId *big.Int) (CidsCid, error) { - return _PDPVerifier.Contract.GetRootCid(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function getPieceLeafCount(uint256 setId, uint256 pieceId) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetPieceLeafCount(setId *big.Int, pieceId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetPieceLeafCount(&_PDPVerifier.CallOpts, setId, pieceId) } -// GetRootCid is a free data retrieval call binding the contract method 0x3b7ae913. +// GetPieceLeafCount is a free data retrieval call binding the contract method 0x0cd7b880. // -// Solidity: function getRootCid(uint256 setId, uint256 rootId) view returns((bytes)) -func (_PDPVerifier *PDPVerifierCallerSession) GetRootCid(setId *big.Int, rootId *big.Int) (CidsCid, error) { - return _PDPVerifier.Contract.GetRootCid(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function getPieceLeafCount(uint256 setId, uint256 pieceId) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetPieceLeafCount(setId *big.Int, pieceId *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetPieceLeafCount(&_PDPVerifier.CallOpts, setId, pieceId) } -// GetRootLeafCount is a free data retrieval call binding the contract method 0x9153e64b. +// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. // -// Solidity: function getRootLeafCount(uint256 setId, uint256 rootId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) GetRootLeafCount(opts *bind.CallOpts, setId *big.Int, rootId *big.Int) (*big.Int, error) { +// Solidity: function getRandomness(uint256 epoch) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) GetRandomness(opts *bind.CallOpts, epoch *big.Int) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getRootLeafCount", setId, rootId) + err := _PDPVerifier.contract.Call(opts, &out, "getRandomness", epoch) if err != nil { return *new(*big.Int), err @@ -1028,18 +1145,18 @@ func (_PDPVerifier *PDPVerifierCaller) GetRootLeafCount(opts *bind.CallOpts, set } -// GetRootLeafCount is a free data retrieval call binding the contract method 0x9153e64b. +// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. // -// Solidity: function getRootLeafCount(uint256 setId, uint256 rootId) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) GetRootLeafCount(setId *big.Int, rootId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetRootLeafCount(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function getRandomness(uint256 epoch) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) GetRandomness(epoch *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetRandomness(&_PDPVerifier.CallOpts, epoch) } -// GetRootLeafCount is a free data retrieval call binding the contract method 0x9153e64b. +// GetRandomness is a free data retrieval call binding the contract method 0x453f4f62. // -// Solidity: function getRootLeafCount(uint256 setId, uint256 rootId) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) GetRootLeafCount(setId *big.Int, rootId *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.GetRootLeafCount(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function getRandomness(uint256 epoch) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) GetRandomness(epoch *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.GetRandomness(&_PDPVerifier.CallOpts, epoch) } // GetScheduledRemovals is a free data retrieval call binding the contract method 0x6fa44692. @@ -1104,12 +1221,12 @@ func (_PDPVerifier *PDPVerifierCallerSession) Owner() (common.Address, error) { return _PDPVerifier.Contract.Owner(&_PDPVerifier.CallOpts) } -// ProofSetLive is a free data retrieval call binding the contract method 0xf5cac1ba. +// PieceChallengable is a free data retrieval call binding the contract method 0xdc635266. // -// Solidity: function proofSetLive(uint256 setId) view returns(bool) -func (_PDPVerifier *PDPVerifierCaller) ProofSetLive(opts *bind.CallOpts, setId *big.Int) (bool, error) { +// Solidity: function pieceChallengable(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierCaller) PieceChallengable(opts *bind.CallOpts, setId *big.Int, pieceId *big.Int) (bool, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "proofSetLive", setId) + err := _PDPVerifier.contract.Call(opts, &out, "pieceChallengable", setId, pieceId) if err != nil { return *new(bool), err @@ -1121,57 +1238,26 @@ func (_PDPVerifier *PDPVerifierCaller) ProofSetLive(opts *bind.CallOpts, setId * } -// ProofSetLive is a free data retrieval call binding the contract method 0xf5cac1ba. -// -// Solidity: function proofSetLive(uint256 setId) view returns(bool) -func (_PDPVerifier *PDPVerifierSession) ProofSetLive(setId *big.Int) (bool, error) { - return _PDPVerifier.Contract.ProofSetLive(&_PDPVerifier.CallOpts, setId) -} - -// ProofSetLive is a free data retrieval call binding the contract method 0xf5cac1ba. -// -// Solidity: function proofSetLive(uint256 setId) view returns(bool) -func (_PDPVerifier *PDPVerifierCallerSession) ProofSetLive(setId *big.Int) (bool, error) { - return _PDPVerifier.Contract.ProofSetLive(&_PDPVerifier.CallOpts, setId) -} - -// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. -// -// Solidity: function proxiableUUID() view returns(bytes32) -func (_PDPVerifier *PDPVerifierCaller) ProxiableUUID(opts *bind.CallOpts) ([32]byte, error) { - var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "proxiableUUID") - - if err != nil { - return *new([32]byte), err - } - - out0 := *abi.ConvertType(out[0], new([32]byte)).(*[32]byte) - - return out0, err - -} - -// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. +// PieceChallengable is a free data retrieval call binding the contract method 0xdc635266. // -// Solidity: function proxiableUUID() view returns(bytes32) -func (_PDPVerifier *PDPVerifierSession) ProxiableUUID() ([32]byte, error) { - return _PDPVerifier.Contract.ProxiableUUID(&_PDPVerifier.CallOpts) +// Solidity: function pieceChallengable(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierSession) PieceChallengable(setId *big.Int, pieceId *big.Int) (bool, error) { + return _PDPVerifier.Contract.PieceChallengable(&_PDPVerifier.CallOpts, setId, pieceId) } -// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. +// PieceChallengable is a free data retrieval call binding the contract method 0xdc635266. // -// Solidity: function proxiableUUID() view returns(bytes32) -func (_PDPVerifier *PDPVerifierCallerSession) ProxiableUUID() ([32]byte, error) { - return _PDPVerifier.Contract.ProxiableUUID(&_PDPVerifier.CallOpts) +// Solidity: function pieceChallengable(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierCallerSession) PieceChallengable(setId *big.Int, pieceId *big.Int) (bool, error) { + return _PDPVerifier.Contract.PieceChallengable(&_PDPVerifier.CallOpts, setId, pieceId) } -// RootChallengable is a free data retrieval call binding the contract method 0x71cf2a16. +// PieceLive is a free data retrieval call binding the contract method 0x1a271225. // -// Solidity: function rootChallengable(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierCaller) RootChallengable(opts *bind.CallOpts, setId *big.Int, rootId *big.Int) (bool, error) { +// Solidity: function pieceLive(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierCaller) PieceLive(opts *bind.CallOpts, setId *big.Int, pieceId *big.Int) (bool, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "rootChallengable", setId, rootId) + err := _PDPVerifier.contract.Call(opts, &out, "pieceLive", setId, pieceId) if err != nil { return *new(bool), err @@ -1183,133 +1269,133 @@ func (_PDPVerifier *PDPVerifierCaller) RootChallengable(opts *bind.CallOpts, set } -// RootChallengable is a free data retrieval call binding the contract method 0x71cf2a16. +// PieceLive is a free data retrieval call binding the contract method 0x1a271225. // -// Solidity: function rootChallengable(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierSession) RootChallengable(setId *big.Int, rootId *big.Int) (bool, error) { - return _PDPVerifier.Contract.RootChallengable(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function pieceLive(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierSession) PieceLive(setId *big.Int, pieceId *big.Int) (bool, error) { + return _PDPVerifier.Contract.PieceLive(&_PDPVerifier.CallOpts, setId, pieceId) } -// RootChallengable is a free data retrieval call binding the contract method 0x71cf2a16. +// PieceLive is a free data retrieval call binding the contract method 0x1a271225. // -// Solidity: function rootChallengable(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierCallerSession) RootChallengable(setId *big.Int, rootId *big.Int) (bool, error) { - return _PDPVerifier.Contract.RootChallengable(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function pieceLive(uint256 setId, uint256 pieceId) view returns(bool) +func (_PDPVerifier *PDPVerifierCallerSession) PieceLive(setId *big.Int, pieceId *big.Int) (bool, error) { + return _PDPVerifier.Contract.PieceLive(&_PDPVerifier.CallOpts, setId, pieceId) } -// RootLive is a free data retrieval call binding the contract method 0x47331050. +// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. // -// Solidity: function rootLive(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierCaller) RootLive(opts *bind.CallOpts, setId *big.Int, rootId *big.Int) (bool, error) { +// Solidity: function proxiableUUID() view returns(bytes32) +func (_PDPVerifier *PDPVerifierCaller) ProxiableUUID(opts *bind.CallOpts) ([32]byte, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "rootLive", setId, rootId) + err := _PDPVerifier.contract.Call(opts, &out, "proxiableUUID") if err != nil { - return *new(bool), err + return *new([32]byte), err } - out0 := *abi.ConvertType(out[0], new(bool)).(*bool) + out0 := *abi.ConvertType(out[0], new([32]byte)).(*[32]byte) return out0, err } -// RootLive is a free data retrieval call binding the contract method 0x47331050. +// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. // -// Solidity: function rootLive(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierSession) RootLive(setId *big.Int, rootId *big.Int) (bool, error) { - return _PDPVerifier.Contract.RootLive(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function proxiableUUID() view returns(bytes32) +func (_PDPVerifier *PDPVerifierSession) ProxiableUUID() ([32]byte, error) { + return _PDPVerifier.Contract.ProxiableUUID(&_PDPVerifier.CallOpts) } -// RootLive is a free data retrieval call binding the contract method 0x47331050. +// ProxiableUUID is a free data retrieval call binding the contract method 0x52d1902d. // -// Solidity: function rootLive(uint256 setId, uint256 rootId) view returns(bool) -func (_PDPVerifier *PDPVerifierCallerSession) RootLive(setId *big.Int, rootId *big.Int) (bool, error) { - return _PDPVerifier.Contract.RootLive(&_PDPVerifier.CallOpts, setId, rootId) +// Solidity: function proxiableUUID() view returns(bytes32) +func (_PDPVerifier *PDPVerifierCallerSession) ProxiableUUID() ([32]byte, error) { + return _PDPVerifier.Contract.ProxiableUUID(&_PDPVerifier.CallOpts) } -// AddRoots is a paid mutator transaction binding the contract method 0x11c0ee4a. +// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. // -// Solidity: function addRoots(uint256 setId, ((bytes),uint256)[] rootData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactor) AddRoots(opts *bind.TransactOpts, setId *big.Int, rootData []PDPVerifierRootData, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "addRoots", setId, rootData, extraData) +// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactor) AddPieces(opts *bind.TransactOpts, setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "addPieces", setId, pieceData, extraData) } -// AddRoots is a paid mutator transaction binding the contract method 0x11c0ee4a. +// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. // -// Solidity: function addRoots(uint256 setId, ((bytes),uint256)[] rootData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierSession) AddRoots(setId *big.Int, rootData []PDPVerifierRootData, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.AddRoots(&_PDPVerifier.TransactOpts, setId, rootData, extraData) +// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierSession) AddPieces(setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.AddPieces(&_PDPVerifier.TransactOpts, setId, pieceData, extraData) } -// AddRoots is a paid mutator transaction binding the contract method 0x11c0ee4a. +// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. // -// Solidity: function addRoots(uint256 setId, ((bytes),uint256)[] rootData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactorSession) AddRoots(setId *big.Int, rootData []PDPVerifierRootData, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.AddRoots(&_PDPVerifier.TransactOpts, setId, rootData, extraData) +// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactorSession) AddPieces(setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.AddPieces(&_PDPVerifier.TransactOpts, setId, pieceData, extraData) } -// ClaimProofSetOwnership is a paid mutator transaction binding the contract method 0xee3dac65. +// ClaimDataSetStorageProvider is a paid mutator transaction binding the contract method 0xdf0f3248. // -// Solidity: function claimProofSetOwnership(uint256 setId) returns() -func (_PDPVerifier *PDPVerifierTransactor) ClaimProofSetOwnership(opts *bind.TransactOpts, setId *big.Int) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "claimProofSetOwnership", setId) +// Solidity: function claimDataSetStorageProvider(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactor) ClaimDataSetStorageProvider(opts *bind.TransactOpts, setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "claimDataSetStorageProvider", setId, extraData) } -// ClaimProofSetOwnership is a paid mutator transaction binding the contract method 0xee3dac65. +// ClaimDataSetStorageProvider is a paid mutator transaction binding the contract method 0xdf0f3248. // -// Solidity: function claimProofSetOwnership(uint256 setId) returns() -func (_PDPVerifier *PDPVerifierSession) ClaimProofSetOwnership(setId *big.Int) (*types.Transaction, error) { - return _PDPVerifier.Contract.ClaimProofSetOwnership(&_PDPVerifier.TransactOpts, setId) +// Solidity: function claimDataSetStorageProvider(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierSession) ClaimDataSetStorageProvider(setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.ClaimDataSetStorageProvider(&_PDPVerifier.TransactOpts, setId, extraData) } -// ClaimProofSetOwnership is a paid mutator transaction binding the contract method 0xee3dac65. +// ClaimDataSetStorageProvider is a paid mutator transaction binding the contract method 0xdf0f3248. // -// Solidity: function claimProofSetOwnership(uint256 setId) returns() -func (_PDPVerifier *PDPVerifierTransactorSession) ClaimProofSetOwnership(setId *big.Int) (*types.Transaction, error) { - return _PDPVerifier.Contract.ClaimProofSetOwnership(&_PDPVerifier.TransactOpts, setId) +// Solidity: function claimDataSetStorageProvider(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactorSession) ClaimDataSetStorageProvider(setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.ClaimDataSetStorageProvider(&_PDPVerifier.TransactOpts, setId, extraData) } -// CreateProofSet is a paid mutator transaction binding the contract method 0x0a4d7932. +// CreateDataSet is a paid mutator transaction binding the contract method 0xbbae41cb. // -// Solidity: function createProofSet(address listenerAddr, bytes extraData) payable returns(uint256) -func (_PDPVerifier *PDPVerifierTransactor) CreateProofSet(opts *bind.TransactOpts, listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "createProofSet", listenerAddr, extraData) +// Solidity: function createDataSet(address listenerAddr, bytes extraData) payable returns(uint256) +func (_PDPVerifier *PDPVerifierTransactor) CreateDataSet(opts *bind.TransactOpts, listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "createDataSet", listenerAddr, extraData) } -// CreateProofSet is a paid mutator transaction binding the contract method 0x0a4d7932. +// CreateDataSet is a paid mutator transaction binding the contract method 0xbbae41cb. // -// Solidity: function createProofSet(address listenerAddr, bytes extraData) payable returns(uint256) -func (_PDPVerifier *PDPVerifierSession) CreateProofSet(listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.CreateProofSet(&_PDPVerifier.TransactOpts, listenerAddr, extraData) +// Solidity: function createDataSet(address listenerAddr, bytes extraData) payable returns(uint256) +func (_PDPVerifier *PDPVerifierSession) CreateDataSet(listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.CreateDataSet(&_PDPVerifier.TransactOpts, listenerAddr, extraData) } -// CreateProofSet is a paid mutator transaction binding the contract method 0x0a4d7932. +// CreateDataSet is a paid mutator transaction binding the contract method 0xbbae41cb. // -// Solidity: function createProofSet(address listenerAddr, bytes extraData) payable returns(uint256) -func (_PDPVerifier *PDPVerifierTransactorSession) CreateProofSet(listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.CreateProofSet(&_PDPVerifier.TransactOpts, listenerAddr, extraData) +// Solidity: function createDataSet(address listenerAddr, bytes extraData) payable returns(uint256) +func (_PDPVerifier *PDPVerifierTransactorSession) CreateDataSet(listenerAddr common.Address, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.CreateDataSet(&_PDPVerifier.TransactOpts, listenerAddr, extraData) } -// DeleteProofSet is a paid mutator transaction binding the contract method 0x847d1d06. +// DeleteDataSet is a paid mutator transaction binding the contract method 0x7a1e2990. // -// Solidity: function deleteProofSet(uint256 setId, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierTransactor) DeleteProofSet(opts *bind.TransactOpts, setId *big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "deleteProofSet", setId, extraData) +// Solidity: function deleteDataSet(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactor) DeleteDataSet(opts *bind.TransactOpts, setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "deleteDataSet", setId, extraData) } -// DeleteProofSet is a paid mutator transaction binding the contract method 0x847d1d06. +// DeleteDataSet is a paid mutator transaction binding the contract method 0x7a1e2990. // -// Solidity: function deleteProofSet(uint256 setId, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierSession) DeleteProofSet(setId *big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.DeleteProofSet(&_PDPVerifier.TransactOpts, setId, extraData) +// Solidity: function deleteDataSet(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierSession) DeleteDataSet(setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.DeleteDataSet(&_PDPVerifier.TransactOpts, setId, extraData) } -// DeleteProofSet is a paid mutator transaction binding the contract method 0x847d1d06. +// DeleteDataSet is a paid mutator transaction binding the contract method 0x7a1e2990. // -// Solidity: function deleteProofSet(uint256 setId, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierTransactorSession) DeleteProofSet(setId *big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.DeleteProofSet(&_PDPVerifier.TransactOpts, setId, extraData) +// Solidity: function deleteDataSet(uint256 setId, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactorSession) DeleteDataSet(setId *big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.DeleteDataSet(&_PDPVerifier.TransactOpts, setId, extraData) } // Initialize is a paid mutator transaction binding the contract method 0xfe4b84df. @@ -1354,25 +1440,25 @@ func (_PDPVerifier *PDPVerifierTransactorSession) NextProvingPeriod(setId *big.I return _PDPVerifier.Contract.NextProvingPeriod(&_PDPVerifier.TransactOpts, setId, challengeEpoch, extraData) } -// ProposeProofSetOwner is a paid mutator transaction binding the contract method 0x6cb55c16. +// ProposeDataSetStorageProvider is a paid mutator transaction binding the contract method 0x43186080. // -// Solidity: function proposeProofSetOwner(uint256 setId, address newOwner) returns() -func (_PDPVerifier *PDPVerifierTransactor) ProposeProofSetOwner(opts *bind.TransactOpts, setId *big.Int, newOwner common.Address) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "proposeProofSetOwner", setId, newOwner) +// Solidity: function proposeDataSetStorageProvider(uint256 setId, address newStorageProvider) returns() +func (_PDPVerifier *PDPVerifierTransactor) ProposeDataSetStorageProvider(opts *bind.TransactOpts, setId *big.Int, newStorageProvider common.Address) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "proposeDataSetStorageProvider", setId, newStorageProvider) } -// ProposeProofSetOwner is a paid mutator transaction binding the contract method 0x6cb55c16. +// ProposeDataSetStorageProvider is a paid mutator transaction binding the contract method 0x43186080. // -// Solidity: function proposeProofSetOwner(uint256 setId, address newOwner) returns() -func (_PDPVerifier *PDPVerifierSession) ProposeProofSetOwner(setId *big.Int, newOwner common.Address) (*types.Transaction, error) { - return _PDPVerifier.Contract.ProposeProofSetOwner(&_PDPVerifier.TransactOpts, setId, newOwner) +// Solidity: function proposeDataSetStorageProvider(uint256 setId, address newStorageProvider) returns() +func (_PDPVerifier *PDPVerifierSession) ProposeDataSetStorageProvider(setId *big.Int, newStorageProvider common.Address) (*types.Transaction, error) { + return _PDPVerifier.Contract.ProposeDataSetStorageProvider(&_PDPVerifier.TransactOpts, setId, newStorageProvider) } -// ProposeProofSetOwner is a paid mutator transaction binding the contract method 0x6cb55c16. +// ProposeDataSetStorageProvider is a paid mutator transaction binding the contract method 0x43186080. // -// Solidity: function proposeProofSetOwner(uint256 setId, address newOwner) returns() -func (_PDPVerifier *PDPVerifierTransactorSession) ProposeProofSetOwner(setId *big.Int, newOwner common.Address) (*types.Transaction, error) { - return _PDPVerifier.Contract.ProposeProofSetOwner(&_PDPVerifier.TransactOpts, setId, newOwner) +// Solidity: function proposeDataSetStorageProvider(uint256 setId, address newStorageProvider) returns() +func (_PDPVerifier *PDPVerifierTransactorSession) ProposeDataSetStorageProvider(setId *big.Int, newStorageProvider common.Address) (*types.Transaction, error) { + return _PDPVerifier.Contract.ProposeDataSetStorageProvider(&_PDPVerifier.TransactOpts, setId, newStorageProvider) } // ProvePossession is a paid mutator transaction binding the contract method 0xf58f952b. @@ -1417,25 +1503,25 @@ func (_PDPVerifier *PDPVerifierTransactorSession) RenounceOwnership() (*types.Tr return _PDPVerifier.Contract.RenounceOwnership(&_PDPVerifier.TransactOpts) } -// ScheduleRemovals is a paid mutator transaction binding the contract method 0x3b68e4e9. +// SchedulePieceDeletions is a paid mutator transaction binding the contract method 0x0c292024. // -// Solidity: function scheduleRemovals(uint256 setId, uint256[] rootIds, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierTransactor) ScheduleRemovals(opts *bind.TransactOpts, setId *big.Int, rootIds []*big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "scheduleRemovals", setId, rootIds, extraData) +// Solidity: function schedulePieceDeletions(uint256 setId, uint256[] pieceIds, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactor) SchedulePieceDeletions(opts *bind.TransactOpts, setId *big.Int, pieceIds []*big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "schedulePieceDeletions", setId, pieceIds, extraData) } -// ScheduleRemovals is a paid mutator transaction binding the contract method 0x3b68e4e9. +// SchedulePieceDeletions is a paid mutator transaction binding the contract method 0x0c292024. // -// Solidity: function scheduleRemovals(uint256 setId, uint256[] rootIds, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierSession) ScheduleRemovals(setId *big.Int, rootIds []*big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.ScheduleRemovals(&_PDPVerifier.TransactOpts, setId, rootIds, extraData) +// Solidity: function schedulePieceDeletions(uint256 setId, uint256[] pieceIds, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierSession) SchedulePieceDeletions(setId *big.Int, pieceIds []*big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.SchedulePieceDeletions(&_PDPVerifier.TransactOpts, setId, pieceIds, extraData) } -// ScheduleRemovals is a paid mutator transaction binding the contract method 0x3b68e4e9. +// SchedulePieceDeletions is a paid mutator transaction binding the contract method 0x0c292024. // -// Solidity: function scheduleRemovals(uint256 setId, uint256[] rootIds, bytes extraData) returns() -func (_PDPVerifier *PDPVerifierTransactorSession) ScheduleRemovals(setId *big.Int, rootIds []*big.Int, extraData []byte) (*types.Transaction, error) { - return _PDPVerifier.Contract.ScheduleRemovals(&_PDPVerifier.TransactOpts, setId, rootIds, extraData) +// Solidity: function schedulePieceDeletions(uint256 setId, uint256[] pieceIds, bytes extraData) returns() +func (_PDPVerifier *PDPVerifierTransactorSession) SchedulePieceDeletions(setId *big.Int, pieceIds []*big.Int, extraData []byte) (*types.Transaction, error) { + return _PDPVerifier.Contract.SchedulePieceDeletions(&_PDPVerifier.TransactOpts, setId, pieceIds, extraData) } // TransferOwnership is a paid mutator transaction binding the contract method 0xf2fde38b. @@ -1480,9 +1566,9 @@ func (_PDPVerifier *PDPVerifierTransactorSession) UpgradeToAndCall(newImplementa return _PDPVerifier.Contract.UpgradeToAndCall(&_PDPVerifier.TransactOpts, newImplementation, data) } -// PDPVerifierDebugIterator is returned from FilterDebug and is used to iterate over the raw logs and unpacked data for Debug events raised by the PDPVerifier contract. -type PDPVerifierDebugIterator struct { - Event *PDPVerifierDebug // Event containing the contract specifics and raw log +// PDPVerifierDataSetCreatedIterator is returned from FilterDataSetCreated and is used to iterate over the raw logs and unpacked data for DataSetCreated events raised by the PDPVerifier contract. +type PDPVerifierDataSetCreatedIterator struct { + Event *PDPVerifierDataSetCreated // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1496,7 +1582,7 @@ type PDPVerifierDebugIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierDebugIterator) Next() bool { +func (it *PDPVerifierDataSetCreatedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1505,7 +1591,7 @@ func (it *PDPVerifierDebugIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierDebug) + it.Event = new(PDPVerifierDataSetCreated) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1520,7 +1606,7 @@ func (it *PDPVerifierDebugIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierDebug) + it.Event = new(PDPVerifierDataSetCreated) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1536,43 +1622,61 @@ func (it *PDPVerifierDebugIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierDebugIterator) Error() error { +func (it *PDPVerifierDataSetCreatedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierDebugIterator) Close() error { +func (it *PDPVerifierDataSetCreatedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierDebug represents a Debug event raised by the PDPVerifier contract. -type PDPVerifierDebug struct { - Message string - Value *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetCreated represents a DataSetCreated event raised by the PDPVerifier contract. +type PDPVerifierDataSetCreated struct { + SetId *big.Int + StorageProvider common.Address + Raw types.Log // Blockchain specific contextual infos } -// FilterDebug is a free log retrieval operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// FilterDataSetCreated is a free log retrieval operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) FilterDebug(opts *bind.FilterOpts) (*PDPVerifierDebugIterator, error) { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetCreated(opts *bind.FilterOpts, setId []*big.Int, storageProvider []common.Address) (*PDPVerifierDataSetCreatedIterator, error) { - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "Debug") + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } + var storageProviderRule []interface{} + for _, storageProviderItem := range storageProvider { + storageProviderRule = append(storageProviderRule, storageProviderItem) + } + + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) if err != nil { return nil, err } - return &PDPVerifierDebugIterator{contract: _PDPVerifier.contract, event: "Debug", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetCreatedIterator{contract: _PDPVerifier.contract, event: "DataSetCreated", logs: logs, sub: sub}, nil } -// WatchDebug is a free log subscription operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// WatchDataSetCreated is a free log subscription operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink chan<- *PDPVerifierDebug) (event.Subscription, error) { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetCreated(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetCreated, setId []*big.Int, storageProvider []common.Address) (event.Subscription, error) { - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "Debug") - if err != nil { + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } + var storageProviderRule []interface{} + for _, storageProviderItem := range storageProvider { + storageProviderRule = append(storageProviderRule, storageProviderItem) + } + + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) + if err != nil { return nil, err } return event.NewSubscription(func(quit <-chan struct{}) error { @@ -1581,8 +1685,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink c select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierDebug) - if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { + event := new(PDPVerifierDataSetCreated) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { return err } event.Raw = log @@ -1603,21 +1707,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink c }), nil } -// ParseDebug is a log parse operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// ParseDataSetCreated is a log parse operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) ParseDebug(log types.Log) (*PDPVerifierDebug, error) { - event := new(PDPVerifierDebug) - if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetCreated(log types.Log) (*PDPVerifierDataSetCreated, error) { + event := new(PDPVerifierDataSetCreated) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierInitializedIterator is returned from FilterInitialized and is used to iterate over the raw logs and unpacked data for Initialized events raised by the PDPVerifier contract. -type PDPVerifierInitializedIterator struct { - Event *PDPVerifierInitialized // Event containing the contract specifics and raw log +// PDPVerifierDataSetDeletedIterator is returned from FilterDataSetDeleted and is used to iterate over the raw logs and unpacked data for DataSetDeleted events raised by the PDPVerifier contract. +type PDPVerifierDataSetDeletedIterator struct { + Event *PDPVerifierDataSetDeleted // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1631,7 +1735,7 @@ type PDPVerifierInitializedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierInitializedIterator) Next() bool { +func (it *PDPVerifierDataSetDeletedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1640,7 +1744,7 @@ func (it *PDPVerifierInitializedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierInitialized) + it.Event = new(PDPVerifierDataSetDeleted) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1655,7 +1759,7 @@ func (it *PDPVerifierInitializedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierInitialized) + it.Event = new(PDPVerifierDataSetDeleted) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1671,41 +1775,52 @@ func (it *PDPVerifierInitializedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierInitializedIterator) Error() error { +func (it *PDPVerifierDataSetDeletedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierInitializedIterator) Close() error { +func (it *PDPVerifierDataSetDeletedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierInitialized represents a Initialized event raised by the PDPVerifier contract. -type PDPVerifierInitialized struct { - Version uint64 - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetDeleted represents a DataSetDeleted event raised by the PDPVerifier contract. +type PDPVerifierDataSetDeleted struct { + SetId *big.Int + DeletedLeafCount *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterInitialized is a free log retrieval operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. +// FilterDataSetDeleted is a free log retrieval operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event Initialized(uint64 version) -func (_PDPVerifier *PDPVerifierFilterer) FilterInitialized(opts *bind.FilterOpts) (*PDPVerifierInitializedIterator, error) { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetDeleted(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetDeletedIterator, error) { - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "Initialized") + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } + + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetDeleted", setIdRule) if err != nil { return nil, err } - return &PDPVerifierInitializedIterator{contract: _PDPVerifier.contract, event: "Initialized", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetDeletedIterator{contract: _PDPVerifier.contract, event: "DataSetDeleted", logs: logs, sub: sub}, nil } -// WatchInitialized is a free log subscription operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. +// WatchDataSetDeleted is a free log subscription operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event Initialized(uint64 version) -func (_PDPVerifier *PDPVerifierFilterer) WatchInitialized(opts *bind.WatchOpts, sink chan<- *PDPVerifierInitialized) (event.Subscription, error) { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetDeleted(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetDeleted, setId []*big.Int) (event.Subscription, error) { - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "Initialized") + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } + + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetDeleted", setIdRule) if err != nil { return nil, err } @@ -1715,8 +1830,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchInitialized(opts *bind.WatchOpts, select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierInitialized) - if err := _PDPVerifier.contract.UnpackLog(event, "Initialized", log); err != nil { + event := new(PDPVerifierDataSetDeleted) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { return err } event.Raw = log @@ -1737,21 +1852,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchInitialized(opts *bind.WatchOpts, }), nil } -// ParseInitialized is a log parse operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. +// ParseDataSetDeleted is a log parse operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event Initialized(uint64 version) -func (_PDPVerifier *PDPVerifierFilterer) ParseInitialized(log types.Log) (*PDPVerifierInitialized, error) { - event := new(PDPVerifierInitialized) - if err := _PDPVerifier.contract.UnpackLog(event, "Initialized", log); err != nil { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetDeleted(log types.Log) (*PDPVerifierDataSetDeleted, error) { + event := new(PDPVerifierDataSetDeleted) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierNextProvingPeriodIterator is returned from FilterNextProvingPeriod and is used to iterate over the raw logs and unpacked data for NextProvingPeriod events raised by the PDPVerifier contract. -type PDPVerifierNextProvingPeriodIterator struct { - Event *PDPVerifierNextProvingPeriod // Event containing the contract specifics and raw log +// PDPVerifierDataSetEmptyIterator is returned from FilterDataSetEmpty and is used to iterate over the raw logs and unpacked data for DataSetEmpty events raised by the PDPVerifier contract. +type PDPVerifierDataSetEmptyIterator struct { + Event *PDPVerifierDataSetEmpty // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1765,7 +1880,7 @@ type PDPVerifierNextProvingPeriodIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierNextProvingPeriodIterator) Next() bool { +func (it *PDPVerifierDataSetEmptyIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1774,7 +1889,7 @@ func (it *PDPVerifierNextProvingPeriodIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierNextProvingPeriod) + it.Event = new(PDPVerifierDataSetEmpty) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1789,7 +1904,7 @@ func (it *PDPVerifierNextProvingPeriodIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierNextProvingPeriod) + it.Event = new(PDPVerifierDataSetEmpty) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1805,53 +1920,51 @@ func (it *PDPVerifierNextProvingPeriodIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierNextProvingPeriodIterator) Error() error { +func (it *PDPVerifierDataSetEmptyIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierNextProvingPeriodIterator) Close() error { +func (it *PDPVerifierDataSetEmptyIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierNextProvingPeriod represents a NextProvingPeriod event raised by the PDPVerifier contract. -type PDPVerifierNextProvingPeriod struct { - SetId *big.Int - ChallengeEpoch *big.Int - LeafCount *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetEmpty represents a DataSetEmpty event raised by the PDPVerifier contract. +type PDPVerifierDataSetEmpty struct { + SetId *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterNextProvingPeriod is a free log retrieval operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. +// FilterDataSetEmpty is a free log retrieval operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) -func (_PDPVerifier *PDPVerifierFilterer) FilterNextProvingPeriod(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierNextProvingPeriodIterator, error) { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetEmpty(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetEmptyIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "NextProvingPeriod", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetEmpty", setIdRule) if err != nil { return nil, err } - return &PDPVerifierNextProvingPeriodIterator{contract: _PDPVerifier.contract, event: "NextProvingPeriod", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetEmptyIterator{contract: _PDPVerifier.contract, event: "DataSetEmpty", logs: logs, sub: sub}, nil } -// WatchNextProvingPeriod is a free log subscription operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. +// WatchDataSetEmpty is a free log subscription operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) -func (_PDPVerifier *PDPVerifierFilterer) WatchNextProvingPeriod(opts *bind.WatchOpts, sink chan<- *PDPVerifierNextProvingPeriod, setId []*big.Int) (event.Subscription, error) { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetEmpty(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetEmpty, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "NextProvingPeriod", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetEmpty", setIdRule) if err != nil { return nil, err } @@ -1861,8 +1974,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchNextProvingPeriod(opts *bind.Watch select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierNextProvingPeriod) - if err := _PDPVerifier.contract.UnpackLog(event, "NextProvingPeriod", log); err != nil { + event := new(PDPVerifierDataSetEmpty) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { return err } event.Raw = log @@ -1883,21 +1996,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchNextProvingPeriod(opts *bind.Watch }), nil } -// ParseNextProvingPeriod is a log parse operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. +// ParseDataSetEmpty is a log parse operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) -func (_PDPVerifier *PDPVerifierFilterer) ParseNextProvingPeriod(log types.Log) (*PDPVerifierNextProvingPeriod, error) { - event := new(PDPVerifierNextProvingPeriod) - if err := _PDPVerifier.contract.UnpackLog(event, "NextProvingPeriod", log); err != nil { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetEmpty(log types.Log) (*PDPVerifierDataSetEmpty, error) { + event := new(PDPVerifierDataSetEmpty) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierOwnershipTransferredIterator is returned from FilterOwnershipTransferred and is used to iterate over the raw logs and unpacked data for OwnershipTransferred events raised by the PDPVerifier contract. -type PDPVerifierOwnershipTransferredIterator struct { - Event *PDPVerifierOwnershipTransferred // Event containing the contract specifics and raw log +// PDPVerifierDebugIterator is returned from FilterDebug and is used to iterate over the raw logs and unpacked data for Debug events raised by the PDPVerifier contract. +type PDPVerifierDebugIterator struct { + Event *PDPVerifierDebug // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1911,7 +2024,7 @@ type PDPVerifierOwnershipTransferredIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierOwnershipTransferredIterator) Next() bool { +func (it *PDPVerifierDebugIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1920,7 +2033,7 @@ func (it *PDPVerifierOwnershipTransferredIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierOwnershipTransferred) + it.Event = new(PDPVerifierDebug) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1935,7 +2048,7 @@ func (it *PDPVerifierOwnershipTransferredIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierOwnershipTransferred) + it.Event = new(PDPVerifierDebug) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1951,60 +2064,42 @@ func (it *PDPVerifierOwnershipTransferredIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierOwnershipTransferredIterator) Error() error { +func (it *PDPVerifierDebugIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierOwnershipTransferredIterator) Close() error { +func (it *PDPVerifierDebugIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierOwnershipTransferred represents a OwnershipTransferred event raised by the PDPVerifier contract. -type PDPVerifierOwnershipTransferred struct { - PreviousOwner common.Address - NewOwner common.Address - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDebug represents a Debug event raised by the PDPVerifier contract. +type PDPVerifierDebug struct { + Message string + Value *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterOwnershipTransferred is a free log retrieval operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. +// FilterDebug is a free log retrieval operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. // -// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) FilterOwnershipTransferred(opts *bind.FilterOpts, previousOwner []common.Address, newOwner []common.Address) (*PDPVerifierOwnershipTransferredIterator, error) { - - var previousOwnerRule []interface{} - for _, previousOwnerItem := range previousOwner { - previousOwnerRule = append(previousOwnerRule, previousOwnerItem) - } - var newOwnerRule []interface{} - for _, newOwnerItem := range newOwner { - newOwnerRule = append(newOwnerRule, newOwnerItem) - } +// Solidity: event Debug(string message, uint256 value) +func (_PDPVerifier *PDPVerifierFilterer) FilterDebug(opts *bind.FilterOpts) (*PDPVerifierDebugIterator, error) { - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "OwnershipTransferred", previousOwnerRule, newOwnerRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "Debug") if err != nil { return nil, err } - return &PDPVerifierOwnershipTransferredIterator{contract: _PDPVerifier.contract, event: "OwnershipTransferred", logs: logs, sub: sub}, nil + return &PDPVerifierDebugIterator{contract: _PDPVerifier.contract, event: "Debug", logs: logs, sub: sub}, nil } -// WatchOwnershipTransferred is a free log subscription operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. +// WatchDebug is a free log subscription operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. // -// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) WatchOwnershipTransferred(opts *bind.WatchOpts, sink chan<- *PDPVerifierOwnershipTransferred, previousOwner []common.Address, newOwner []common.Address) (event.Subscription, error) { - - var previousOwnerRule []interface{} - for _, previousOwnerItem := range previousOwner { - previousOwnerRule = append(previousOwnerRule, previousOwnerItem) - } - var newOwnerRule []interface{} - for _, newOwnerItem := range newOwner { - newOwnerRule = append(newOwnerRule, newOwnerItem) - } +// Solidity: event Debug(string message, uint256 value) +func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink chan<- *PDPVerifierDebug) (event.Subscription, error) { - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "OwnershipTransferred", previousOwnerRule, newOwnerRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "Debug") if err != nil { return nil, err } @@ -2014,8 +2109,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchOwnershipTransferred(opts *bind.Wa select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierOwnershipTransferred) - if err := _PDPVerifier.contract.UnpackLog(event, "OwnershipTransferred", log); err != nil { + event := new(PDPVerifierDebug) + if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { return err } event.Raw = log @@ -2036,21 +2131,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchOwnershipTransferred(opts *bind.Wa }), nil } -// ParseOwnershipTransferred is a log parse operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. +// ParseDebug is a log parse operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. // -// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) ParseOwnershipTransferred(log types.Log) (*PDPVerifierOwnershipTransferred, error) { - event := new(PDPVerifierOwnershipTransferred) - if err := _PDPVerifier.contract.UnpackLog(event, "OwnershipTransferred", log); err != nil { +// Solidity: event Debug(string message, uint256 value) +func (_PDPVerifier *PDPVerifierFilterer) ParseDebug(log types.Log) (*PDPVerifierDebug, error) { + event := new(PDPVerifierDebug) + if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierPossessionProvenIterator is returned from FilterPossessionProven and is used to iterate over the raw logs and unpacked data for PossessionProven events raised by the PDPVerifier contract. -type PDPVerifierPossessionProvenIterator struct { - Event *PDPVerifierPossessionProven // Event containing the contract specifics and raw log +// PDPVerifierInitializedIterator is returned from FilterInitialized and is used to iterate over the raw logs and unpacked data for Initialized events raised by the PDPVerifier contract. +type PDPVerifierInitializedIterator struct { + Event *PDPVerifierInitialized // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2064,7 +2159,7 @@ type PDPVerifierPossessionProvenIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierPossessionProvenIterator) Next() bool { +func (it *PDPVerifierInitializedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2073,7 +2168,7 @@ func (it *PDPVerifierPossessionProvenIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierPossessionProven) + it.Event = new(PDPVerifierInitialized) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2088,7 +2183,7 @@ func (it *PDPVerifierPossessionProvenIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierPossessionProven) + it.Event = new(PDPVerifierInitialized) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2104,52 +2199,41 @@ func (it *PDPVerifierPossessionProvenIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierPossessionProvenIterator) Error() error { +func (it *PDPVerifierInitializedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierPossessionProvenIterator) Close() error { +func (it *PDPVerifierInitializedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierPossessionProven represents a PossessionProven event raised by the PDPVerifier contract. -type PDPVerifierPossessionProven struct { - SetId *big.Int - Challenges []PDPVerifierRootIdAndOffset - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierInitialized represents a Initialized event raised by the PDPVerifier contract. +type PDPVerifierInitialized struct { + Version uint64 + Raw types.Log // Blockchain specific contextual infos } -// FilterPossessionProven is a free log retrieval operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. +// FilterInitialized is a free log retrieval operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. // -// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) -func (_PDPVerifier *PDPVerifierFilterer) FilterPossessionProven(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierPossessionProvenIterator, error) { - - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) - } +// Solidity: event Initialized(uint64 version) +func (_PDPVerifier *PDPVerifierFilterer) FilterInitialized(opts *bind.FilterOpts) (*PDPVerifierInitializedIterator, error) { - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PossessionProven", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "Initialized") if err != nil { return nil, err } - return &PDPVerifierPossessionProvenIterator{contract: _PDPVerifier.contract, event: "PossessionProven", logs: logs, sub: sub}, nil + return &PDPVerifierInitializedIterator{contract: _PDPVerifier.contract, event: "Initialized", logs: logs, sub: sub}, nil } -// WatchPossessionProven is a free log subscription operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. +// WatchInitialized is a free log subscription operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. // -// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) -func (_PDPVerifier *PDPVerifierFilterer) WatchPossessionProven(opts *bind.WatchOpts, sink chan<- *PDPVerifierPossessionProven, setId []*big.Int) (event.Subscription, error) { - - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) - } +// Solidity: event Initialized(uint64 version) +func (_PDPVerifier *PDPVerifierFilterer) WatchInitialized(opts *bind.WatchOpts, sink chan<- *PDPVerifierInitialized) (event.Subscription, error) { - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PossessionProven", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "Initialized") if err != nil { return nil, err } @@ -2159,8 +2243,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchPossessionProven(opts *bind.WatchO select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierPossessionProven) - if err := _PDPVerifier.contract.UnpackLog(event, "PossessionProven", log); err != nil { + event := new(PDPVerifierInitialized) + if err := _PDPVerifier.contract.UnpackLog(event, "Initialized", log); err != nil { return err } event.Raw = log @@ -2181,21 +2265,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchPossessionProven(opts *bind.WatchO }), nil } -// ParsePossessionProven is a log parse operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. +// ParseInitialized is a log parse operation binding the contract event 0xc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2. // -// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) -func (_PDPVerifier *PDPVerifierFilterer) ParsePossessionProven(log types.Log) (*PDPVerifierPossessionProven, error) { - event := new(PDPVerifierPossessionProven) - if err := _PDPVerifier.contract.UnpackLog(event, "PossessionProven", log); err != nil { +// Solidity: event Initialized(uint64 version) +func (_PDPVerifier *PDPVerifierFilterer) ParseInitialized(log types.Log) (*PDPVerifierInitialized, error) { + event := new(PDPVerifierInitialized) + if err := _PDPVerifier.contract.UnpackLog(event, "Initialized", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierProofFeePaidIterator is returned from FilterProofFeePaid and is used to iterate over the raw logs and unpacked data for ProofFeePaid events raised by the PDPVerifier contract. -type PDPVerifierProofFeePaidIterator struct { - Event *PDPVerifierProofFeePaid // Event containing the contract specifics and raw log +// PDPVerifierNextProvingPeriodIterator is returned from FilterNextProvingPeriod and is used to iterate over the raw logs and unpacked data for NextProvingPeriod events raised by the PDPVerifier contract. +type PDPVerifierNextProvingPeriodIterator struct { + Event *PDPVerifierNextProvingPeriod // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2209,7 +2293,7 @@ type PDPVerifierProofFeePaidIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierProofFeePaidIterator) Next() bool { +func (it *PDPVerifierNextProvingPeriodIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2218,7 +2302,7 @@ func (it *PDPVerifierProofFeePaidIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierProofFeePaid) + it.Event = new(PDPVerifierNextProvingPeriod) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2233,7 +2317,7 @@ func (it *PDPVerifierProofFeePaidIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierProofFeePaid) + it.Event = new(PDPVerifierNextProvingPeriod) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2249,54 +2333,53 @@ func (it *PDPVerifierProofFeePaidIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierProofFeePaidIterator) Error() error { +func (it *PDPVerifierNextProvingPeriodIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierProofFeePaidIterator) Close() error { +func (it *PDPVerifierNextProvingPeriodIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierProofFeePaid represents a ProofFeePaid event raised by the PDPVerifier contract. -type PDPVerifierProofFeePaid struct { - SetId *big.Int - Fee *big.Int - Price uint64 - Expo int32 - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierNextProvingPeriod represents a NextProvingPeriod event raised by the PDPVerifier contract. +type PDPVerifierNextProvingPeriod struct { + SetId *big.Int + ChallengeEpoch *big.Int + LeafCount *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterProofFeePaid is a free log retrieval operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. +// FilterNextProvingPeriod is a free log retrieval operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. // -// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) -func (_PDPVerifier *PDPVerifierFilterer) FilterProofFeePaid(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierProofFeePaidIterator, error) { +// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) +func (_PDPVerifier *PDPVerifierFilterer) FilterNextProvingPeriod(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierNextProvingPeriodIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofFeePaid", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "NextProvingPeriod", setIdRule) if err != nil { return nil, err } - return &PDPVerifierProofFeePaidIterator{contract: _PDPVerifier.contract, event: "ProofFeePaid", logs: logs, sub: sub}, nil + return &PDPVerifierNextProvingPeriodIterator{contract: _PDPVerifier.contract, event: "NextProvingPeriod", logs: logs, sub: sub}, nil } -// WatchProofFeePaid is a free log subscription operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. +// WatchNextProvingPeriod is a free log subscription operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. // -// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) -func (_PDPVerifier *PDPVerifierFilterer) WatchProofFeePaid(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofFeePaid, setId []*big.Int) (event.Subscription, error) { +// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) +func (_PDPVerifier *PDPVerifierFilterer) WatchNextProvingPeriod(opts *bind.WatchOpts, sink chan<- *PDPVerifierNextProvingPeriod, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofFeePaid", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "NextProvingPeriod", setIdRule) if err != nil { return nil, err } @@ -2306,8 +2389,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofFeePaid(opts *bind.WatchOpts, select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierProofFeePaid) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofFeePaid", log); err != nil { + event := new(PDPVerifierNextProvingPeriod) + if err := _PDPVerifier.contract.UnpackLog(event, "NextProvingPeriod", log); err != nil { return err } event.Raw = log @@ -2328,21 +2411,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofFeePaid(opts *bind.WatchOpts, }), nil } -// ParseProofFeePaid is a log parse operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. +// ParseNextProvingPeriod is a log parse operation binding the contract event 0xc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e. // -// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) -func (_PDPVerifier *PDPVerifierFilterer) ParseProofFeePaid(log types.Log) (*PDPVerifierProofFeePaid, error) { - event := new(PDPVerifierProofFeePaid) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofFeePaid", log); err != nil { +// Solidity: event NextProvingPeriod(uint256 indexed setId, uint256 challengeEpoch, uint256 leafCount) +func (_PDPVerifier *PDPVerifierFilterer) ParseNextProvingPeriod(log types.Log) (*PDPVerifierNextProvingPeriod, error) { + event := new(PDPVerifierNextProvingPeriod) + if err := _PDPVerifier.contract.UnpackLog(event, "NextProvingPeriod", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierProofSetCreatedIterator is returned from FilterProofSetCreated and is used to iterate over the raw logs and unpacked data for ProofSetCreated events raised by the PDPVerifier contract. -type PDPVerifierProofSetCreatedIterator struct { - Event *PDPVerifierProofSetCreated // Event containing the contract specifics and raw log +// PDPVerifierOwnershipTransferredIterator is returned from FilterOwnershipTransferred and is used to iterate over the raw logs and unpacked data for OwnershipTransferred events raised by the PDPVerifier contract. +type PDPVerifierOwnershipTransferredIterator struct { + Event *PDPVerifierOwnershipTransferred // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2356,7 +2439,7 @@ type PDPVerifierProofSetCreatedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierProofSetCreatedIterator) Next() bool { +func (it *PDPVerifierOwnershipTransferredIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2365,7 +2448,7 @@ func (it *PDPVerifierProofSetCreatedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetCreated) + it.Event = new(PDPVerifierOwnershipTransferred) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2380,7 +2463,7 @@ func (it *PDPVerifierProofSetCreatedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetCreated) + it.Event = new(PDPVerifierOwnershipTransferred) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2396,60 +2479,60 @@ func (it *PDPVerifierProofSetCreatedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierProofSetCreatedIterator) Error() error { +func (it *PDPVerifierOwnershipTransferredIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierProofSetCreatedIterator) Close() error { +func (it *PDPVerifierOwnershipTransferredIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierProofSetCreated represents a ProofSetCreated event raised by the PDPVerifier contract. -type PDPVerifierProofSetCreated struct { - SetId *big.Int - Owner common.Address - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierOwnershipTransferred represents a OwnershipTransferred event raised by the PDPVerifier contract. +type PDPVerifierOwnershipTransferred struct { + PreviousOwner common.Address + NewOwner common.Address + Raw types.Log // Blockchain specific contextual infos } -// FilterProofSetCreated is a free log retrieval operation binding the contract event 0x017f0b33d96e8f9968590172013032c2346cf047787a5e17a44b0a1bb3cd0f01. +// FilterOwnershipTransferred is a free log retrieval operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. // -// Solidity: event ProofSetCreated(uint256 indexed setId, address indexed owner) -func (_PDPVerifier *PDPVerifierFilterer) FilterProofSetCreated(opts *bind.FilterOpts, setId []*big.Int, owner []common.Address) (*PDPVerifierProofSetCreatedIterator, error) { +// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) +func (_PDPVerifier *PDPVerifierFilterer) FilterOwnershipTransferred(opts *bind.FilterOpts, previousOwner []common.Address, newOwner []common.Address) (*PDPVerifierOwnershipTransferredIterator, error) { - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) + var previousOwnerRule []interface{} + for _, previousOwnerItem := range previousOwner { + previousOwnerRule = append(previousOwnerRule, previousOwnerItem) } - var ownerRule []interface{} - for _, ownerItem := range owner { - ownerRule = append(ownerRule, ownerItem) + var newOwnerRule []interface{} + for _, newOwnerItem := range newOwner { + newOwnerRule = append(newOwnerRule, newOwnerItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofSetCreated", setIdRule, ownerRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "OwnershipTransferred", previousOwnerRule, newOwnerRule) if err != nil { return nil, err } - return &PDPVerifierProofSetCreatedIterator{contract: _PDPVerifier.contract, event: "ProofSetCreated", logs: logs, sub: sub}, nil + return &PDPVerifierOwnershipTransferredIterator{contract: _PDPVerifier.contract, event: "OwnershipTransferred", logs: logs, sub: sub}, nil } -// WatchProofSetCreated is a free log subscription operation binding the contract event 0x017f0b33d96e8f9968590172013032c2346cf047787a5e17a44b0a1bb3cd0f01. +// WatchOwnershipTransferred is a free log subscription operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. // -// Solidity: event ProofSetCreated(uint256 indexed setId, address indexed owner) -func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetCreated(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofSetCreated, setId []*big.Int, owner []common.Address) (event.Subscription, error) { +// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) +func (_PDPVerifier *PDPVerifierFilterer) WatchOwnershipTransferred(opts *bind.WatchOpts, sink chan<- *PDPVerifierOwnershipTransferred, previousOwner []common.Address, newOwner []common.Address) (event.Subscription, error) { - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) + var previousOwnerRule []interface{} + for _, previousOwnerItem := range previousOwner { + previousOwnerRule = append(previousOwnerRule, previousOwnerItem) } - var ownerRule []interface{} - for _, ownerItem := range owner { - ownerRule = append(ownerRule, ownerItem) + var newOwnerRule []interface{} + for _, newOwnerItem := range newOwner { + newOwnerRule = append(newOwnerRule, newOwnerItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofSetCreated", setIdRule, ownerRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "OwnershipTransferred", previousOwnerRule, newOwnerRule) if err != nil { return nil, err } @@ -2459,8 +2542,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetCreated(opts *bind.WatchOp select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierProofSetCreated) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetCreated", log); err != nil { + event := new(PDPVerifierOwnershipTransferred) + if err := _PDPVerifier.contract.UnpackLog(event, "OwnershipTransferred", log); err != nil { return err } event.Raw = log @@ -2481,21 +2564,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetCreated(opts *bind.WatchOp }), nil } -// ParseProofSetCreated is a log parse operation binding the contract event 0x017f0b33d96e8f9968590172013032c2346cf047787a5e17a44b0a1bb3cd0f01. +// ParseOwnershipTransferred is a log parse operation binding the contract event 0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0. // -// Solidity: event ProofSetCreated(uint256 indexed setId, address indexed owner) -func (_PDPVerifier *PDPVerifierFilterer) ParseProofSetCreated(log types.Log) (*PDPVerifierProofSetCreated, error) { - event := new(PDPVerifierProofSetCreated) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetCreated", log); err != nil { +// Solidity: event OwnershipTransferred(address indexed previousOwner, address indexed newOwner) +func (_PDPVerifier *PDPVerifierFilterer) ParseOwnershipTransferred(log types.Log) (*PDPVerifierOwnershipTransferred, error) { + event := new(PDPVerifierOwnershipTransferred) + if err := _PDPVerifier.contract.UnpackLog(event, "OwnershipTransferred", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierProofSetDeletedIterator is returned from FilterProofSetDeleted and is used to iterate over the raw logs and unpacked data for ProofSetDeleted events raised by the PDPVerifier contract. -type PDPVerifierProofSetDeletedIterator struct { - Event *PDPVerifierProofSetDeleted // Event containing the contract specifics and raw log +// PDPVerifierPiecesAddedIterator is returned from FilterPiecesAdded and is used to iterate over the raw logs and unpacked data for PiecesAdded events raised by the PDPVerifier contract. +type PDPVerifierPiecesAddedIterator struct { + Event *PDPVerifierPiecesAdded // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2509,7 +2592,7 @@ type PDPVerifierProofSetDeletedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierProofSetDeletedIterator) Next() bool { +func (it *PDPVerifierPiecesAddedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2518,7 +2601,7 @@ func (it *PDPVerifierProofSetDeletedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetDeleted) + it.Event = new(PDPVerifierPiecesAdded) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2533,7 +2616,7 @@ func (it *PDPVerifierProofSetDeletedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetDeleted) + it.Event = new(PDPVerifierPiecesAdded) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2549,52 +2632,52 @@ func (it *PDPVerifierProofSetDeletedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierProofSetDeletedIterator) Error() error { +func (it *PDPVerifierPiecesAddedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierProofSetDeletedIterator) Close() error { +func (it *PDPVerifierPiecesAddedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierProofSetDeleted represents a ProofSetDeleted event raised by the PDPVerifier contract. -type PDPVerifierProofSetDeleted struct { - SetId *big.Int - DeletedLeafCount *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierPiecesAdded represents a PiecesAdded event raised by the PDPVerifier contract. +type PDPVerifierPiecesAdded struct { + SetId *big.Int + PieceIds []*big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterProofSetDeleted is a free log retrieval operation binding the contract event 0x589e9a441b5bddda77c4ab647b0108764a9cc1a7f655aa9b7bc50b8bdfab8673. +// FilterPiecesAdded is a free log retrieval operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. // -// Solidity: event ProofSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) FilterProofSetDeleted(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierProofSetDeletedIterator, error) { +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) FilterPiecesAdded(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierPiecesAddedIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofSetDeleted", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PiecesAdded", setIdRule) if err != nil { return nil, err } - return &PDPVerifierProofSetDeletedIterator{contract: _PDPVerifier.contract, event: "ProofSetDeleted", logs: logs, sub: sub}, nil + return &PDPVerifierPiecesAddedIterator{contract: _PDPVerifier.contract, event: "PiecesAdded", logs: logs, sub: sub}, nil } -// WatchProofSetDeleted is a free log subscription operation binding the contract event 0x589e9a441b5bddda77c4ab647b0108764a9cc1a7f655aa9b7bc50b8bdfab8673. +// WatchPiecesAdded is a free log subscription operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. // -// Solidity: event ProofSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetDeleted(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofSetDeleted, setId []*big.Int) (event.Subscription, error) { +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) WatchPiecesAdded(opts *bind.WatchOpts, sink chan<- *PDPVerifierPiecesAdded, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofSetDeleted", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PiecesAdded", setIdRule) if err != nil { return nil, err } @@ -2604,8 +2687,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetDeleted(opts *bind.WatchOp select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierProofSetDeleted) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetDeleted", log); err != nil { + event := new(PDPVerifierPiecesAdded) + if err := _PDPVerifier.contract.UnpackLog(event, "PiecesAdded", log); err != nil { return err } event.Raw = log @@ -2626,21 +2709,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetDeleted(opts *bind.WatchOp }), nil } -// ParseProofSetDeleted is a log parse operation binding the contract event 0x589e9a441b5bddda77c4ab647b0108764a9cc1a7f655aa9b7bc50b8bdfab8673. +// ParsePiecesAdded is a log parse operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. // -// Solidity: event ProofSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) ParseProofSetDeleted(log types.Log) (*PDPVerifierProofSetDeleted, error) { - event := new(PDPVerifierProofSetDeleted) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetDeleted", log); err != nil { +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) ParsePiecesAdded(log types.Log) (*PDPVerifierPiecesAdded, error) { + event := new(PDPVerifierPiecesAdded) + if err := _PDPVerifier.contract.UnpackLog(event, "PiecesAdded", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierProofSetEmptyIterator is returned from FilterProofSetEmpty and is used to iterate over the raw logs and unpacked data for ProofSetEmpty events raised by the PDPVerifier contract. -type PDPVerifierProofSetEmptyIterator struct { - Event *PDPVerifierProofSetEmpty // Event containing the contract specifics and raw log +// PDPVerifierPiecesRemovedIterator is returned from FilterPiecesRemoved and is used to iterate over the raw logs and unpacked data for PiecesRemoved events raised by the PDPVerifier contract. +type PDPVerifierPiecesRemovedIterator struct { + Event *PDPVerifierPiecesRemoved // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2654,7 +2737,7 @@ type PDPVerifierProofSetEmptyIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierProofSetEmptyIterator) Next() bool { +func (it *PDPVerifierPiecesRemovedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2663,7 +2746,7 @@ func (it *PDPVerifierProofSetEmptyIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetEmpty) + it.Event = new(PDPVerifierPiecesRemoved) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2678,7 +2761,7 @@ func (it *PDPVerifierProofSetEmptyIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetEmpty) + it.Event = new(PDPVerifierPiecesRemoved) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2694,51 +2777,52 @@ func (it *PDPVerifierProofSetEmptyIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierProofSetEmptyIterator) Error() error { +func (it *PDPVerifierPiecesRemovedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierProofSetEmptyIterator) Close() error { +func (it *PDPVerifierPiecesRemovedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierProofSetEmpty represents a ProofSetEmpty event raised by the PDPVerifier contract. -type PDPVerifierProofSetEmpty struct { - SetId *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierPiecesRemoved represents a PiecesRemoved event raised by the PDPVerifier contract. +type PDPVerifierPiecesRemoved struct { + SetId *big.Int + PieceIds []*big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterProofSetEmpty is a free log retrieval operation binding the contract event 0x323c29bc8d678a5d987b90a321982d10b9a91bcad071a9e445879497bf0e68e7. +// FilterPiecesRemoved is a free log retrieval operation binding the contract event 0x6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b1. // -// Solidity: event ProofSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) FilterProofSetEmpty(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierProofSetEmptyIterator, error) { +// Solidity: event PiecesRemoved(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) FilterPiecesRemoved(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierPiecesRemovedIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofSetEmpty", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PiecesRemoved", setIdRule) if err != nil { return nil, err } - return &PDPVerifierProofSetEmptyIterator{contract: _PDPVerifier.contract, event: "ProofSetEmpty", logs: logs, sub: sub}, nil + return &PDPVerifierPiecesRemovedIterator{contract: _PDPVerifier.contract, event: "PiecesRemoved", logs: logs, sub: sub}, nil } -// WatchProofSetEmpty is a free log subscription operation binding the contract event 0x323c29bc8d678a5d987b90a321982d10b9a91bcad071a9e445879497bf0e68e7. +// WatchPiecesRemoved is a free log subscription operation binding the contract event 0x6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b1. // -// Solidity: event ProofSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetEmpty(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofSetEmpty, setId []*big.Int) (event.Subscription, error) { +// Solidity: event PiecesRemoved(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) WatchPiecesRemoved(opts *bind.WatchOpts, sink chan<- *PDPVerifierPiecesRemoved, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofSetEmpty", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PiecesRemoved", setIdRule) if err != nil { return nil, err } @@ -2748,8 +2832,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetEmpty(opts *bind.WatchOpts select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierProofSetEmpty) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetEmpty", log); err != nil { + event := new(PDPVerifierPiecesRemoved) + if err := _PDPVerifier.contract.UnpackLog(event, "PiecesRemoved", log); err != nil { return err } event.Raw = log @@ -2770,21 +2854,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetEmpty(opts *bind.WatchOpts }), nil } -// ParseProofSetEmpty is a log parse operation binding the contract event 0x323c29bc8d678a5d987b90a321982d10b9a91bcad071a9e445879497bf0e68e7. +// ParsePiecesRemoved is a log parse operation binding the contract event 0x6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b1. // -// Solidity: event ProofSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) ParseProofSetEmpty(log types.Log) (*PDPVerifierProofSetEmpty, error) { - event := new(PDPVerifierProofSetEmpty) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetEmpty", log); err != nil { +// Solidity: event PiecesRemoved(uint256 indexed setId, uint256[] pieceIds) +func (_PDPVerifier *PDPVerifierFilterer) ParsePiecesRemoved(log types.Log) (*PDPVerifierPiecesRemoved, error) { + event := new(PDPVerifierPiecesRemoved) + if err := _PDPVerifier.contract.UnpackLog(event, "PiecesRemoved", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierProofSetOwnerChangedIterator is returned from FilterProofSetOwnerChanged and is used to iterate over the raw logs and unpacked data for ProofSetOwnerChanged events raised by the PDPVerifier contract. -type PDPVerifierProofSetOwnerChangedIterator struct { - Event *PDPVerifierProofSetOwnerChanged // Event containing the contract specifics and raw log +// PDPVerifierPossessionProvenIterator is returned from FilterPossessionProven and is used to iterate over the raw logs and unpacked data for PossessionProven events raised by the PDPVerifier contract. +type PDPVerifierPossessionProvenIterator struct { + Event *PDPVerifierPossessionProven // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2798,7 +2882,7 @@ type PDPVerifierProofSetOwnerChangedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierProofSetOwnerChangedIterator) Next() bool { +func (it *PDPVerifierPossessionProvenIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2807,7 +2891,7 @@ func (it *PDPVerifierProofSetOwnerChangedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetOwnerChanged) + it.Event = new(PDPVerifierPossessionProven) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2822,7 +2906,7 @@ func (it *PDPVerifierProofSetOwnerChangedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierProofSetOwnerChanged) + it.Event = new(PDPVerifierPossessionProven) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2838,69 +2922,52 @@ func (it *PDPVerifierProofSetOwnerChangedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierProofSetOwnerChangedIterator) Error() error { +func (it *PDPVerifierPossessionProvenIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierProofSetOwnerChangedIterator) Close() error { +func (it *PDPVerifierPossessionProvenIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierProofSetOwnerChanged represents a ProofSetOwnerChanged event raised by the PDPVerifier contract. -type PDPVerifierProofSetOwnerChanged struct { - SetId *big.Int - OldOwner common.Address - NewOwner common.Address - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierPossessionProven represents a PossessionProven event raised by the PDPVerifier contract. +type PDPVerifierPossessionProven struct { + SetId *big.Int + Challenges []PDPVerifierPieceIdAndOffset + Raw types.Log // Blockchain specific contextual infos } -// FilterProofSetOwnerChanged is a free log retrieval operation binding the contract event 0xd3273037b635678293ef0c076bd77af13760e75e12806d1db237616d03c3a766. +// FilterPossessionProven is a free log retrieval operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. // -// Solidity: event ProofSetOwnerChanged(uint256 indexed setId, address indexed oldOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) FilterProofSetOwnerChanged(opts *bind.FilterOpts, setId []*big.Int, oldOwner []common.Address, newOwner []common.Address) (*PDPVerifierProofSetOwnerChangedIterator, error) { +// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) +func (_PDPVerifier *PDPVerifierFilterer) FilterPossessionProven(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierPossessionProvenIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - var oldOwnerRule []interface{} - for _, oldOwnerItem := range oldOwner { - oldOwnerRule = append(oldOwnerRule, oldOwnerItem) - } - var newOwnerRule []interface{} - for _, newOwnerItem := range newOwner { - newOwnerRule = append(newOwnerRule, newOwnerItem) - } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofSetOwnerChanged", setIdRule, oldOwnerRule, newOwnerRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PossessionProven", setIdRule) if err != nil { return nil, err } - return &PDPVerifierProofSetOwnerChangedIterator{contract: _PDPVerifier.contract, event: "ProofSetOwnerChanged", logs: logs, sub: sub}, nil + return &PDPVerifierPossessionProvenIterator{contract: _PDPVerifier.contract, event: "PossessionProven", logs: logs, sub: sub}, nil } -// WatchProofSetOwnerChanged is a free log subscription operation binding the contract event 0xd3273037b635678293ef0c076bd77af13760e75e12806d1db237616d03c3a766. +// WatchPossessionProven is a free log subscription operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. // -// Solidity: event ProofSetOwnerChanged(uint256 indexed setId, address indexed oldOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetOwnerChanged(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofSetOwnerChanged, setId []*big.Int, oldOwner []common.Address, newOwner []common.Address) (event.Subscription, error) { +// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) +func (_PDPVerifier *PDPVerifierFilterer) WatchPossessionProven(opts *bind.WatchOpts, sink chan<- *PDPVerifierPossessionProven, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - var oldOwnerRule []interface{} - for _, oldOwnerItem := range oldOwner { - oldOwnerRule = append(oldOwnerRule, oldOwnerItem) - } - var newOwnerRule []interface{} - for _, newOwnerItem := range newOwner { - newOwnerRule = append(newOwnerRule, newOwnerItem) - } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofSetOwnerChanged", setIdRule, oldOwnerRule, newOwnerRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PossessionProven", setIdRule) if err != nil { return nil, err } @@ -2910,8 +2977,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetOwnerChanged(opts *bind.Wa select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierProofSetOwnerChanged) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetOwnerChanged", log); err != nil { + event := new(PDPVerifierPossessionProven) + if err := _PDPVerifier.contract.UnpackLog(event, "PossessionProven", log); err != nil { return err } event.Raw = log @@ -2932,21 +2999,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchProofSetOwnerChanged(opts *bind.Wa }), nil } -// ParseProofSetOwnerChanged is a log parse operation binding the contract event 0xd3273037b635678293ef0c076bd77af13760e75e12806d1db237616d03c3a766. +// ParsePossessionProven is a log parse operation binding the contract event 0x1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e50. // -// Solidity: event ProofSetOwnerChanged(uint256 indexed setId, address indexed oldOwner, address indexed newOwner) -func (_PDPVerifier *PDPVerifierFilterer) ParseProofSetOwnerChanged(log types.Log) (*PDPVerifierProofSetOwnerChanged, error) { - event := new(PDPVerifierProofSetOwnerChanged) - if err := _PDPVerifier.contract.UnpackLog(event, "ProofSetOwnerChanged", log); err != nil { +// Solidity: event PossessionProven(uint256 indexed setId, (uint256,uint256)[] challenges) +func (_PDPVerifier *PDPVerifierFilterer) ParsePossessionProven(log types.Log) (*PDPVerifierPossessionProven, error) { + event := new(PDPVerifierPossessionProven) + if err := _PDPVerifier.contract.UnpackLog(event, "PossessionProven", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierRootsAddedIterator is returned from FilterRootsAdded and is used to iterate over the raw logs and unpacked data for RootsAdded events raised by the PDPVerifier contract. -type PDPVerifierRootsAddedIterator struct { - Event *PDPVerifierRootsAdded // Event containing the contract specifics and raw log +// PDPVerifierProofFeePaidIterator is returned from FilterProofFeePaid and is used to iterate over the raw logs and unpacked data for ProofFeePaid events raised by the PDPVerifier contract. +type PDPVerifierProofFeePaidIterator struct { + Event *PDPVerifierProofFeePaid // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2960,7 +3027,7 @@ type PDPVerifierRootsAddedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierRootsAddedIterator) Next() bool { +func (it *PDPVerifierProofFeePaidIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2969,7 +3036,7 @@ func (it *PDPVerifierRootsAddedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierRootsAdded) + it.Event = new(PDPVerifierProofFeePaid) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2984,7 +3051,7 @@ func (it *PDPVerifierRootsAddedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierRootsAdded) + it.Event = new(PDPVerifierProofFeePaid) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -3000,52 +3067,54 @@ func (it *PDPVerifierRootsAddedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierRootsAddedIterator) Error() error { +func (it *PDPVerifierProofFeePaidIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierRootsAddedIterator) Close() error { +func (it *PDPVerifierProofFeePaidIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierRootsAdded represents a RootsAdded event raised by the PDPVerifier contract. -type PDPVerifierRootsAdded struct { - SetId *big.Int - RootIds []*big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierProofFeePaid represents a ProofFeePaid event raised by the PDPVerifier contract. +type PDPVerifierProofFeePaid struct { + SetId *big.Int + Fee *big.Int + Price uint64 + Expo int32 + Raw types.Log // Blockchain specific contextual infos } -// FilterRootsAdded is a free log retrieval operation binding the contract event 0x5ce51a8003915c377679ba533d9dafa0792058b254965697e674272f13f4fdd3. +// FilterProofFeePaid is a free log retrieval operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. // -// Solidity: event RootsAdded(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) FilterRootsAdded(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierRootsAddedIterator, error) { +// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) +func (_PDPVerifier *PDPVerifierFilterer) FilterProofFeePaid(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierProofFeePaidIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "RootsAdded", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ProofFeePaid", setIdRule) if err != nil { return nil, err } - return &PDPVerifierRootsAddedIterator{contract: _PDPVerifier.contract, event: "RootsAdded", logs: logs, sub: sub}, nil + return &PDPVerifierProofFeePaidIterator{contract: _PDPVerifier.contract, event: "ProofFeePaid", logs: logs, sub: sub}, nil } -// WatchRootsAdded is a free log subscription operation binding the contract event 0x5ce51a8003915c377679ba533d9dafa0792058b254965697e674272f13f4fdd3. +// WatchProofFeePaid is a free log subscription operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. // -// Solidity: event RootsAdded(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) WatchRootsAdded(opts *bind.WatchOpts, sink chan<- *PDPVerifierRootsAdded, setId []*big.Int) (event.Subscription, error) { +// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) +func (_PDPVerifier *PDPVerifierFilterer) WatchProofFeePaid(opts *bind.WatchOpts, sink chan<- *PDPVerifierProofFeePaid, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "RootsAdded", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ProofFeePaid", setIdRule) if err != nil { return nil, err } @@ -3055,8 +3124,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchRootsAdded(opts *bind.WatchOpts, s select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierRootsAdded) - if err := _PDPVerifier.contract.UnpackLog(event, "RootsAdded", log); err != nil { + event := new(PDPVerifierProofFeePaid) + if err := _PDPVerifier.contract.UnpackLog(event, "ProofFeePaid", log); err != nil { return err } event.Raw = log @@ -3077,21 +3146,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchRootsAdded(opts *bind.WatchOpts, s }), nil } -// ParseRootsAdded is a log parse operation binding the contract event 0x5ce51a8003915c377679ba533d9dafa0792058b254965697e674272f13f4fdd3. +// ParseProofFeePaid is a log parse operation binding the contract event 0x928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b. // -// Solidity: event RootsAdded(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) ParseRootsAdded(log types.Log) (*PDPVerifierRootsAdded, error) { - event := new(PDPVerifierRootsAdded) - if err := _PDPVerifier.contract.UnpackLog(event, "RootsAdded", log); err != nil { +// Solidity: event ProofFeePaid(uint256 indexed setId, uint256 fee, uint64 price, int32 expo) +func (_PDPVerifier *PDPVerifierFilterer) ParseProofFeePaid(log types.Log) (*PDPVerifierProofFeePaid, error) { + event := new(PDPVerifierProofFeePaid) + if err := _PDPVerifier.contract.UnpackLog(event, "ProofFeePaid", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierRootsRemovedIterator is returned from FilterRootsRemoved and is used to iterate over the raw logs and unpacked data for RootsRemoved events raised by the PDPVerifier contract. -type PDPVerifierRootsRemovedIterator struct { - Event *PDPVerifierRootsRemoved // Event containing the contract specifics and raw log +// PDPVerifierStorageProviderChangedIterator is returned from FilterStorageProviderChanged and is used to iterate over the raw logs and unpacked data for StorageProviderChanged events raised by the PDPVerifier contract. +type PDPVerifierStorageProviderChangedIterator struct { + Event *PDPVerifierStorageProviderChanged // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -3105,7 +3174,7 @@ type PDPVerifierRootsRemovedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierRootsRemovedIterator) Next() bool { +func (it *PDPVerifierStorageProviderChangedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -3114,7 +3183,7 @@ func (it *PDPVerifierRootsRemovedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierRootsRemoved) + it.Event = new(PDPVerifierStorageProviderChanged) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -3129,7 +3198,7 @@ func (it *PDPVerifierRootsRemovedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierRootsRemoved) + it.Event = new(PDPVerifierStorageProviderChanged) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -3145,52 +3214,69 @@ func (it *PDPVerifierRootsRemovedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierRootsRemovedIterator) Error() error { +func (it *PDPVerifierStorageProviderChangedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierRootsRemovedIterator) Close() error { +func (it *PDPVerifierStorageProviderChangedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierRootsRemoved represents a RootsRemoved event raised by the PDPVerifier contract. -type PDPVerifierRootsRemoved struct { - SetId *big.Int - RootIds []*big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierStorageProviderChanged represents a StorageProviderChanged event raised by the PDPVerifier contract. +type PDPVerifierStorageProviderChanged struct { + SetId *big.Int + OldStorageProvider common.Address + NewStorageProvider common.Address + Raw types.Log // Blockchain specific contextual infos } -// FilterRootsRemoved is a free log retrieval operation binding the contract event 0xd22bb0ee05b8ca92312459c76223d3b9bc1bd96fb6c9b18e637ededf92d81174. +// FilterStorageProviderChanged is a free log retrieval operation binding the contract event 0x686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c9. // -// Solidity: event RootsRemoved(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) FilterRootsRemoved(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierRootsRemovedIterator, error) { +// Solidity: event StorageProviderChanged(uint256 indexed setId, address indexed oldStorageProvider, address indexed newStorageProvider) +func (_PDPVerifier *PDPVerifierFilterer) FilterStorageProviderChanged(opts *bind.FilterOpts, setId []*big.Int, oldStorageProvider []common.Address, newStorageProvider []common.Address) (*PDPVerifierStorageProviderChangedIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } + var oldStorageProviderRule []interface{} + for _, oldStorageProviderItem := range oldStorageProvider { + oldStorageProviderRule = append(oldStorageProviderRule, oldStorageProviderItem) + } + var newStorageProviderRule []interface{} + for _, newStorageProviderItem := range newStorageProvider { + newStorageProviderRule = append(newStorageProviderRule, newStorageProviderItem) + } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "RootsRemoved", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "StorageProviderChanged", setIdRule, oldStorageProviderRule, newStorageProviderRule) if err != nil { return nil, err } - return &PDPVerifierRootsRemovedIterator{contract: _PDPVerifier.contract, event: "RootsRemoved", logs: logs, sub: sub}, nil + return &PDPVerifierStorageProviderChangedIterator{contract: _PDPVerifier.contract, event: "StorageProviderChanged", logs: logs, sub: sub}, nil } -// WatchRootsRemoved is a free log subscription operation binding the contract event 0xd22bb0ee05b8ca92312459c76223d3b9bc1bd96fb6c9b18e637ededf92d81174. +// WatchStorageProviderChanged is a free log subscription operation binding the contract event 0x686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c9. // -// Solidity: event RootsRemoved(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) WatchRootsRemoved(opts *bind.WatchOpts, sink chan<- *PDPVerifierRootsRemoved, setId []*big.Int) (event.Subscription, error) { +// Solidity: event StorageProviderChanged(uint256 indexed setId, address indexed oldStorageProvider, address indexed newStorageProvider) +func (_PDPVerifier *PDPVerifierFilterer) WatchStorageProviderChanged(opts *bind.WatchOpts, sink chan<- *PDPVerifierStorageProviderChanged, setId []*big.Int, oldStorageProvider []common.Address, newStorageProvider []common.Address) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } + var oldStorageProviderRule []interface{} + for _, oldStorageProviderItem := range oldStorageProvider { + oldStorageProviderRule = append(oldStorageProviderRule, oldStorageProviderItem) + } + var newStorageProviderRule []interface{} + for _, newStorageProviderItem := range newStorageProvider { + newStorageProviderRule = append(newStorageProviderRule, newStorageProviderItem) + } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "RootsRemoved", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "StorageProviderChanged", setIdRule, oldStorageProviderRule, newStorageProviderRule) if err != nil { return nil, err } @@ -3200,8 +3286,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchRootsRemoved(opts *bind.WatchOpts, select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierRootsRemoved) - if err := _PDPVerifier.contract.UnpackLog(event, "RootsRemoved", log); err != nil { + event := new(PDPVerifierStorageProviderChanged) + if err := _PDPVerifier.contract.UnpackLog(event, "StorageProviderChanged", log); err != nil { return err } event.Raw = log @@ -3222,12 +3308,12 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchRootsRemoved(opts *bind.WatchOpts, }), nil } -// ParseRootsRemoved is a log parse operation binding the contract event 0xd22bb0ee05b8ca92312459c76223d3b9bc1bd96fb6c9b18e637ededf92d81174. +// ParseStorageProviderChanged is a log parse operation binding the contract event 0x686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c9. // -// Solidity: event RootsRemoved(uint256 indexed setId, uint256[] rootIds) -func (_PDPVerifier *PDPVerifierFilterer) ParseRootsRemoved(log types.Log) (*PDPVerifierRootsRemoved, error) { - event := new(PDPVerifierRootsRemoved) - if err := _PDPVerifier.contract.UnpackLog(event, "RootsRemoved", log); err != nil { +// Solidity: event StorageProviderChanged(uint256 indexed setId, address indexed oldStorageProvider, address indexed newStorageProvider) +func (_PDPVerifier *PDPVerifierFilterer) ParseStorageProviderChanged(log types.Log) (*PDPVerifierStorageProviderChanged, error) { + event := new(PDPVerifierStorageProviderChanged) + if err := _PDPVerifier.contract.UnpackLog(event, "StorageProviderChanged", log); err != nil { return nil, err } event.Raw = log diff --git a/pdp/contract/types.go b/pdp/contract/types.go index 613da7996..bb46e71a3 100644 --- a/pdp/contract/types.go +++ b/pdp/contract/types.go @@ -2,8 +2,8 @@ package contract import "math/big" -// RootData matches the Solidity RootData struct -type RootData struct { - Root struct{ Data []byte } +// PieceData matches the Solidity PieceData struct +type PieceData struct { + Piece struct{ Data []byte } RawSize *big.Int } diff --git a/pdp/handlers_upload.go b/pdp/handlers_upload.go index e9eee9529..0049f7a99 100644 --- a/pdp/handlers_upload.go +++ b/pdp/handlers_upload.go @@ -260,7 +260,7 @@ func (p *PDPService) handlePieceUpload(w http.ResponseWriter, r *http.Request) { SELECT piece_cid, notify_url, piece_ref, check_hash_codec, check_hash, check_size FROM pdp_piece_uploads WHERE id = $1 `, uploadUUID.String()).Scan(&pieceCIDStr, ¬ifyURL, &pieceRef, &checkHashName, &checkHash, &checkSize) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { http.Error(w, "Upload UUID not found", http.StatusNotFound) } else { http.Error(w, "Database error", http.StatusInternalServerError) diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 0f7c07992..5956d13a2 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -324,8 +324,8 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return false, xerrors.Errorf("converting advertisement to link: %w", err) } - _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - ad.(cidlink.Link).Cid.String(), adv.ContextID, pi.PieceCID.String(), pi.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, md, pcid2.String(), pi.PieceCID.String(), pi.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), adv.Signature, adv.Entries.String()) if err != nil { diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go index 9f8709a5e..367948fe9 100644 --- a/tasks/indexing/task_pdp_ipni.go +++ b/tasks/indexing/task_pdp_ipni.go @@ -3,7 +3,6 @@ package indexing import ( "bytes" "context" - "crypto/rand" "errors" "fmt" "net/url" @@ -11,6 +10,7 @@ import ( "time" "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/maurl" @@ -37,6 +37,7 @@ import ( "github.com/filecoin-project/curio/market/indexstore" "github.com/filecoin-project/curio/market/ipni/chunker" "github.com/filecoin-project/curio/market/ipni/ipniculib" + "github.com/filecoin-project/curio/market/ipni/types" "github.com/filecoin-project/curio/market/mk20" ) @@ -94,99 +95,109 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return true, nil } - var pi abi.PieceInfo - err = pi.UnmarshalCBOR(bytes.NewReader(task.CtxID)) + var pinfo types.PieceInfo + err = pinfo.UnmarshalCBOR(bytes.NewReader(task.CtxID)) if err != nil { return false, xerrors.Errorf("unmarshaling piece info: %w", err) } - var rawSize abi.UnpaddedPieceSize - err = P.db.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1`, pi.PieceCID.String(), pi.Size).Scan(&rawSize) - if err != nil { - return false, xerrors.Errorf("querying raw size: %w", err) - } + pcid2 := pinfo.PieceCID - pcid2, err := commcidv2.PieceCidV2FromV1(pi.PieceCID, uint64(rawSize)) + pi, err := commcidv2.CommPFromPCidV2(pcid2) if err != nil { - return false, xerrors.Errorf("getting piece CID v2: %w", err) + return false, xerrors.Errorf("getting piece info from piece cid: %w", err) } - reader, _, err := P.cpr.GetSharedPieceReader(ctx, pcid2) - if err != nil { - return false, xerrors.Errorf("getting piece reader from piece park: %w", err) - } + var lnk ipld.Link - defer reader.Close() + if pinfo.Payload { + reader, _, err := P.cpr.GetSharedPieceReader(ctx, pcid2) + if err != nil { + return false, xerrors.Errorf("getting piece reader from piece park: %w", err) + } + + defer reader.Close() - recs := make(chan indexstore.Record, 1) + recs := make(chan indexstore.Record, 1) - var eg errgroup.Group - addFail := make(chan struct{}) - var interrupted bool - var subPieces []mk20.DataSource - chk := chunker.NewInitialChunker() + var eg errgroup.Group + addFail := make(chan struct{}) + var interrupted bool + var subPieces []mk20.DataSource + chk := chunker.NewInitialChunker() - eg.Go(func() error { - defer close(addFail) - for rec := range recs { - serr := chk.Accept(rec.Cid.Hash(), int64(rec.Offset), rec.Size) - if serr != nil { - addFail <- struct{}{} - return serr + eg.Go(func() error { + defer close(addFail) + for rec := range recs { + serr := chk.Accept(rec.Cid.Hash(), int64(rec.Offset), rec.Size) + if serr != nil { + addFail <- struct{}{} + return serr + } } - } - return nil - }) + return nil + }) - id, serr := ulid.Parse(task.ID) - if serr != nil { - return false, xerrors.Errorf("parsing task id: %w", serr) - } - deal, serr := mk20.DealFromDB(ctx, P.db, id) - if serr != nil { - return false, xerrors.Errorf("getting deal from db: %w", serr) - } + id, serr := ulid.Parse(task.ID) + if serr != nil { + return false, xerrors.Errorf("parsing task id: %w", serr) + } + deal, serr := mk20.DealFromDB(ctx, P.db, id) + if serr != nil { + return false, xerrors.Errorf("getting deal from db: %w", serr) + } - if deal.Data.Format.Raw != nil { - return false, xerrors.Errorf("raw data not supported") - } + if deal.Data.Format.Raw != nil { + return false, xerrors.Errorf("raw data not supported") + } - if deal.Data.Format.Car != nil { - _, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) - } + if deal.Data.Format.Car != nil { + _, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) + } - if deal.Data.Format.Aggregate != nil { - if deal.Data.Format.Aggregate.Type > 0 { - subPieces = deal.Data.Format.Aggregate.Sub - _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.Size, subPieces, recs, addFail) + if deal.Data.Format.Aggregate != nil { + if deal.Data.Format.Aggregate.Type > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.PieceInfo().Size, subPieces, recs, addFail) + } } - } - if err != nil { - // Chunking itself failed, stop early - close(recs) // still safe to close, chk.Accept() will exit on channel close - // wait for chk.Accept() goroutine to finish cleanly - _ = eg.Wait() - return false, xerrors.Errorf("chunking failed: %w", err) - } + if err != nil { + // Chunking itself failed, stop early + close(recs) // still safe to close, chk.Accept() will exit on channel close + // wait for chk.Accept() goroutine to finish cleanly + _ = eg.Wait() + return false, xerrors.Errorf("chunking failed: %w", err) + } - // Close the channel - close(recs) + // Close the channel + close(recs) - // Wait till is finished - err = eg.Wait() - if err != nil { - return false, xerrors.Errorf("adding index to chunk (interrupted %t): %w", interrupted, err) - } + // Wait till is finished + err = eg.Wait() + if err != nil { + return false, xerrors.Errorf("adding index to chunk (interrupted %t): %w", interrupted, err) + } - // make sure we still own the task before writing to the database - if !stillOwned() { - return false, nil - } + // make sure we still own the task before writing to the database + if !stillOwned() { + return false, nil + } - lnk, err := chk.Finish(ctx, P.db, pcid2) - if err != nil { - return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + lnk, err = chk.Finish(ctx, P.db, pcid2) + if err != nil { + return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + } + } else { + chk := chunker.NewInitialChunker() + err = chk.Accept(pcid2.Hash(), 0, pi.PayloadSize()) + if err != nil { + return false, xerrors.Errorf("adding index to chunk: %w", err) + } + lnk, err = chk.Finish(ctx, P.db, pcid2) + if err != nil { + return false, xerrors.Errorf("chunking CAR multihash iterator: %w", err) + } } // make sure we still own the task before writing ad chains @@ -201,10 +212,21 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return false, xerrors.Errorf("querying previous head: %w", err) } - mds := metadata.IpfsGatewayHttp{} - md, err := mds.MarshalBinary() - if err != nil { - return false, xerrors.Errorf("marshaling metadata: %w", err) + var md []byte + if pinfo.Payload { + mds := metadata.IpfsGatewayHttp{} + mdb, err := mds.MarshalBinary() + if err != nil { + return false, xerrors.Errorf("marshaling metadata: %w", err) + } + md = mdb + } else { + mds := metadata.FilecoinPieceHttp{} + mdb, err := mds.MarshalBinary() + if err != nil { + return false, xerrors.Errorf("marshaling metadata: %w", err) + } + md = mdb } var privKey []byte @@ -276,8 +298,8 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return false, xerrors.Errorf("converting advertisement to link: %w", err) } - _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - ad.(cidlink.Link).Cid.String(), adv.ContextID, pi.PieceCID.String(), pi.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, md, pcid2.String(), pi.PieceInfo().PieceCID.String(), pi.PieceInfo().Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), adv.Signature, adv.Entries.String()) if err != nil { @@ -332,6 +354,7 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask var stop bool for !stop { var markComplete *string + var markCompletePayload *string taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule @@ -340,19 +363,25 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask ID string `db:"id"` PieceCid string `db:"piece_cid"` Size abi.UnpaddedPieceSize `db:"piece_size"` - RawSize abi.UnpaddedPieceSize `db:"raw_size"` - Index bool `db:"indexing"` + PieceCidV2 string `db:"piece_cid_v2"` Announce bool `db:"announce"` + AnnouncePayload bool `db:"announce_payload"` IndexingCreatedAt time.Time `db:"indexing_created_at"` + Announced bool `db:"announced"` + AnnouncedPayload bool `db:"announced_payload"` } err := tx.Select(&pendings, `SELECT id, - piece_cid, + piece_cid_v2, + piece_cid, piece_size, raw_size, indexing, announce, + announce_payload, + announced, + announced_payload, indexing_created_at FROM pdp_pipeline WHERE indexed = TRUE @@ -368,9 +397,10 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask p := pendings[0] - // Skip IPNI if deal says not to announce or not to index (fast retrievals). If we announce without - // indexing, it will cause issue with retrievals. - if !p.Announce || !p.Index { + // Mark deal is complete if: + // 1. We don't need to announce anything + // 2. Both type of announcements are done + if !(p.Announce && p.AnnouncePayload) || (p.AnnouncePayload && p.AnnouncedPayload) { var n int n, err = tx.Exec(`UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1`, p.ID) @@ -396,21 +426,23 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask } var privKey []byte - err = tx.QueryRow(`SELECT priv_key FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&privKey) + var peerIDStr string + err = tx.QueryRow(`SELECT priv_key, peer_id FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&privKey, &peerIDStr) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("failed to get private libp2p key for PDP: %w", err) } - // generate the ipni provider key - pk, _, err := crypto.GenerateEd25519Key(rand.Reader) + var pkey []byte + + err = tx.QueryRow(`SELECT priv_key FROM eth_keys WHERE role = 'pdp'`).Scan(&pkey) if err != nil { - return false, xerrors.Errorf("failed to generate a new key: %w", err) + return false, xerrors.Errorf("failed to get private eth key for PDP: %w", err) } - privKey, err = crypto.MarshalPrivateKey(pk) + pk, err := crypto.UnmarshalPrivateKey(pkey) if err != nil { - return false, xerrors.Errorf("failed to marshal the private key: %w", err) + return false, xerrors.Errorf("unmarshaling private key: %w", err) } pid, err := peer.IDFromPublicKey(pk.GetPublic()) @@ -426,16 +458,13 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask if n == 0 { return false, xerrors.Errorf("failed to insert the key into db") } - } - pkey, err := crypto.UnmarshalPrivateKey(privKey) - if err != nil { - return false, xerrors.Errorf("unmarshaling private key: %w", err) + peerIDStr = pid.String() } - pid, err := peer.IDFromPublicKey(pkey.GetPublic()) + pid, err := peer.Decode(peerIDStr) if err != nil { - return false, fmt.Errorf("getting peer ID: %w", err) + return false, fmt.Errorf("decoding peer ID: %w", err) } pcid, err := cid.Parse(p.PieceCid) @@ -443,58 +472,115 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask return false, xerrors.Errorf("parsing piece CID: %w", err) } - pi := abi.PieceInfo{ - PieceCID: pcid, - Size: abi.PaddedPieceSize(p.Size), + // If we need to announce payload and haven't done so, then do it first + if p.AnnouncePayload && !p.AnnouncedPayload { + pi := types.PieceInfo{ + PieceCID: pcid, + Payload: true, + } + + b := new(bytes.Buffer) + err = pi.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + ilog.Infof("Another IPNI announce task already present for piece %s and payload %d in deal %s", p.PieceCid, p.AnnouncePayload, p.ID) + stop = false // we found a sector to work on, keep going + markCompletePayload = &p.ID + return false, nil + } + if strings.Contains(err.Error(), "already published") { + ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.ID) + stop = false // we found a sector to work on, keep going + markCompletePayload = &p.ID + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + stop = false + markCompletePayload = &p.ID + + // Return early while commiting so we mark complete for payload announcement + return true, nil } - b := new(bytes.Buffer) - err = pi.MarshalCBOR(b) - if err != nil { - return false, xerrors.Errorf("marshaling piece info: %w", err) + // If we don't need to announce payload, mark it as complete so pipeline does not try that + if !p.AnnouncePayload && !p.AnnouncedPayload { + stop = false + markCompletePayload = &p.ID + // Rerun early without commiting so we mark complete for payload announcement + return false, nil } - _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) - if err != nil { - if harmonydb.IsErrUniqueContraint(err) { - ilog.Infof("Another IPNI announce task already present for piece %s in deal %s", p.PieceCid, p.ID) - // SET "complete" status to true for this deal, so it is not considered next time - markComplete = &p.ID - stop = false // we found a sector to work on, keep going - return true, nil + // If we need to announce piece and haven't done so then do it + if p.Announce && !p.Announced { + pi := types.PieceInfo{ + PieceCID: pcid, + Payload: false, + } + b := new(bytes.Buffer) + err = pi.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) } - if strings.Contains(err.Error(), "already published") { - ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.ID) - // SET "complete" status to true for this deal, so it is not considered next time - markComplete = &p.ID - stop = false // we found a sector to work on, keep going - return false, nil + + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + ilog.Infof("Another IPNI announce task already present for piece %s and payload %d in deal %s", p.PieceCid, p.AnnouncePayload, p.ID) + stop = false // we found a sector to work on, keep going + markComplete = &p.ID + return false, nil + + } + if strings.Contains(err.Error(), "already published") { + ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.ID) + stop = false // we found a sector to work on, keep going + markComplete = &p.ID + return false, nil + + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) } - return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + stop = false + markComplete = &p.ID + + // Return early while commiting so we mark complete for piece announcement + return true, nil } - markComplete = &p.ID - stop = false // we found a task to schedule, keep going - return true, nil + // If we don't need to announce piece, mark it as complete so pipeline does not try that + if !p.Announce && !p.Announced { + stop = false + markComplete = &p.ID + // Rerun early without commiting so we mark complete for payload announcement + return false, nil + } + + return false, xerrors.Errorf("no task to schedule") }) if markComplete != nil { - n, err := P.db.Exec(ctx, `UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1 AND complete = FALSE`, *markComplete) + n, err := P.db.Exec(ctx, `UPDATE pdp_pipeline SET announced = TRUE WHERE id = $1`, *markComplete) if err != nil { - log.Errorf("store IPNI success: updating pipeline: %s", err) + log.Errorf("store IPNI success: updating pipeline: %w", err) } if n != 1 { log.Errorf("store IPNI success: updated %d rows", n) } + } - n, err = P.db.Exec(ctx, `UPDATE market_mk20_deal - SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) - WHERE id = $1;`, *markComplete) + if markCompletePayload != nil { + n, err := P.db.Exec(ctx, `UPDATE pdp_pipeline SET announced_payload = TRUE WHERE id = $1`, *markCompletePayload) if err != nil { - log.Errorf("failed to update market_mk20_deal: %w", err) + log.Errorf("store IPNI success: updating pipeline: %w", err) } if n != 1 { - log.Errorf("expected 1 row to be updated, got %d", n) + log.Errorf("store IPNI success: updated %d rows", n) } } } diff --git a/tasks/pdp/proofset_create_watch.go b/tasks/pdp/data_set_create_watch.go similarity index 65% rename from tasks/pdp/proofset_create_watch.go rename to tasks/pdp/data_set_create_watch.go index 1cdef1d92..d21f602c6 100644 --- a/tasks/pdp/proofset_create_watch.go +++ b/tasks/pdp/data_set_create_watch.go @@ -20,17 +20,17 @@ import ( chainTypes "github.com/filecoin-project/lotus/chain/types" ) -type ProofSetCreate struct { +type DataSetCreate struct { CreateMessageHash string `db:"tx_hash"` ID string `db:"id"` Client string `db:"client"` } -func NewWatcherCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { +func NewWatcherDataSetCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetCreates(ctx, db, ethClient) + err := processPendingDataSetCreates(ctx, db, ethClient) if err != nil { - log.Errorf("Failed to process pending proof set creates: %s", err) + log.Errorf("Failed to process pending data set creates: %s", err) } return nil }); err != nil { @@ -38,28 +38,28 @@ func NewWatcherCreate(db *harmonydb.DB, ethClient *ethclient.Client, pcs *chains } } -func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { - // Query for pdp_proof_set_create entries tx_hash is NOT NULL - var proofSetCreates []ProofSetCreate +func processPendingDataSetCreates(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { + // Query for pdp_data_set_create entries tx_hash is NOT NULL + var dataSetCreates []DataSetCreate - err := db.Select(ctx, &proofSetCreates, ` + err := db.Select(ctx, &dataSetCreates, ` SELECT id, client, tx_hash - FROM pdp_proof_set_create + FROM pdp_data_set_create WHERE tx_hash IS NOT NULL`) if err != nil { - return xerrors.Errorf("failed to select proof set creates: %w", err) + return xerrors.Errorf("failed to select data set creates: %w", err) } - if len(proofSetCreates) == 0 { - // No pending proof set creates + if len(dataSetCreates) == 0 { + // No pending data set creates return nil } - // Process each proof set create - for _, psc := range proofSetCreates { - err := processProofSetCreate(ctx, db, psc, ethClient) + // Process each data set create + for _, dsc := range dataSetCreates { + err := processDataSetCreate(ctx, db, dsc, ethClient) if err != nil { - log.Errorf("Failed to process proof set create for tx %s: %s", psc.CreateMessageHash, err) + log.Errorf("Failed to process data set create for tx %s: %s", dsc.CreateMessageHash, err) continue } } @@ -67,26 +67,26 @@ func processPendingProofSetCreates(ctx context.Context, db *harmonydb.DB, ethCli return nil } -func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCreate, ethClient *ethclient.Client) error { +func processDataSetCreate(ctx context.Context, db *harmonydb.DB, dsc DataSetCreate, ethClient *ethclient.Client) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool err := db.QueryRow(ctx, `SELECT tx_receipt, tx_success FROM message_waits_eth WHERE signed_tx_hash = $1 AND tx_success IS NOT NULL - AND tx_receipt IS NOT NULL`, psc.CreateMessageHash).Scan(&txReceiptJSON, &txSuccess) + AND tx_receipt IS NOT NULL`, dsc.CreateMessageHash).Scan(&txReceiptJSON, &txSuccess) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", psc.CreateMessageHash) + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", dsc.CreateMessageHash) } - return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", psc.CreateMessageHash, err) + return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", dsc.CreateMessageHash, err) } // Unmarshal the tx_receipt JSON into types.Receipt var txReceipt types.Receipt err = json.Unmarshal(txReceiptJSON, &txReceipt) if err != nil { - return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", psc.CreateMessageHash, err) + return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", dsc.CreateMessageHash, err) } // Exit early if transaction executed with failure @@ -99,16 +99,16 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), '{complete}', to_jsonb(true), true ) - WHERE id = $2;`, "Transaction failed", psc.ID) + WHERE id = $2;`, "Transaction failed", dsc.ID) if err != nil { return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) } if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_proof_set_create WHERE id = $1`, psc.ID) + _, err = tx.Exec(`DELETE FROM pdp_data_set_create WHERE id = $1`, dsc.ID) if err != nil { - return false, xerrors.Errorf("failed to delete pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to delete pdp_data_set_create: %w", err) } return true, nil }) @@ -121,21 +121,21 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr return nil } - // Parse the logs to extract the proofSetId - proofSetId, err := extractProofSetIdFromReceipt(&txReceipt) + // Parse the logs to extract the dataSetId + dataSetId, err := extractDataSetIdFromReceipt(&txReceipt) if err != nil { - return xerrors.Errorf("failed to extract proofSetId from receipt for tx %s: %w", psc.CreateMessageHash, err) + return xerrors.Errorf("failed to extract dataSetId from receipt for tx %s: %w", dsc.CreateMessageHash, err) } - // Get the listener address for this proof set from the PDPVerifier contract + // Get the listener address for this data set from the PDPVerifier contract pdpVerifier, err := contract.NewPDPVerifier(contract.ContractAddresses().PDPVerifier, ethClient) if err != nil { return xerrors.Errorf("failed to instantiate PDPVerifier contract: %w", err) } - listenerAddr, err := pdpVerifier.GetProofSetListener(nil, big.NewInt(int64(proofSetId))) + listenerAddr, err := pdpVerifier.GetDataSetListener(nil, big.NewInt(int64(dataSetId))) if err != nil { - return xerrors.Errorf("failed to get listener address for proof set %d: %w", proofSetId, err) + return xerrors.Errorf("failed to get listener address for data set %d: %w", dataSetId, err) } // Get the proving period from the listener @@ -146,10 +146,10 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr } comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`INSERT INTO pdp_proof_set (id, client, proving_period, challenge_window, create_deal_id, create_message_hash) - VALUES ($1, $2, $3, $4, $5, $6)`, proofSetId, psc.Client, provingPeriod, challengeWindow, psc.ID, psc.CreateMessageHash) + n, err := tx.Exec(`INSERT INTO pdp_data_set (id, client, proving_period, challenge_window, create_deal_id, create_message_hash) + VALUES ($1, $2, $3, $4, $5, $6)`, dataSetId, dsc.Client, provingPeriod, challengeWindow, dsc.ID, dsc.CreateMessageHash) if err != nil { - return false, xerrors.Errorf("failed to insert pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to insert pdp_data_set_create: %w", err) } if n != 1 { return false, xerrors.Errorf("expected 1 row to be inserted, got %d", n) @@ -157,7 +157,7 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr n, err = tx.Exec(`UPDATE market_mk20_deal SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) - WHERE id = $1;`, psc.ID) + WHERE id = $1;`, dsc.ID) if err != nil { return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) } @@ -165,9 +165,9 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_proof_set_create WHERE id = $1`, psc.ID) + _, err = tx.Exec(`DELETE FROM pdp_data_set_create WHERE id = $1`, dsc.ID) if err != nil { - return false, xerrors.Errorf("failed to delete pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to delete pdp_data_set_create: %w", err) } return true, nil }) @@ -181,15 +181,15 @@ func processProofSetCreate(ctx context.Context, db *harmonydb.DB, psc ProofSetCr return nil } -func extractProofSetIdFromReceipt(receipt *types.Receipt) (uint64, error) { +func extractDataSetIdFromReceipt(receipt *types.Receipt) (uint64, error) { pdpABI, err := contract.PDPVerifierMetaData.GetAbi() if err != nil { return 0, xerrors.Errorf("failed to get PDP ABI: %w", err) } - event, exists := pdpABI.Events["ProofSetCreated"] + event, exists := pdpABI.Events["DataSetCreated"] if !exists { - return 0, xerrors.Errorf("ProofSetCreated event not found in ABI") + return 0, xerrors.Errorf("DataSetCreated event not found in ABI") } for _, vLog := range receipt.Logs { @@ -203,7 +203,7 @@ func extractProofSetIdFromReceipt(receipt *types.Receipt) (uint64, error) { } } - return 0, xerrors.Errorf("ProofSetCreated event not found in receipt") + return 0, xerrors.Errorf("DataSetCreated event not found in receipt") } func getProvingPeriodChallengeWindow(ctx context.Context, ethClient *ethclient.Client, listenerAddr common.Address) (uint64, uint64, error) { diff --git a/tasks/pdp/proofset_delete_watch.go b/tasks/pdp/data_set_delete_watch.go similarity index 72% rename from tasks/pdp/proofset_delete_watch.go rename to tasks/pdp/data_set_delete_watch.go index b3c094190..c3efa7d36 100644 --- a/tasks/pdp/proofset_delete_watch.go +++ b/tasks/pdp/data_set_delete_watch.go @@ -15,7 +15,7 @@ import ( chainTypes "github.com/filecoin-project/lotus/chain/types" ) -type ProofSetDelete struct { +type DataSetDelete struct { DeleteMessageHash string `db:"tx_hash"` ID string `db:"id"` PID int64 `db:"set_id"` @@ -23,9 +23,9 @@ type ProofSetDelete struct { func NewWatcherDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetDeletes(ctx, db) + err := processPendingDataSetDeletes(ctx, db) if err != nil { - log.Errorf("Failed to process pending proof set creates: %s", err) + log.Errorf("Failed to process pending data set creates: %s", err) } return nil }); err != nil { @@ -33,28 +33,28 @@ func NewWatcherDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { } } -func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB) error { - // Query for pdp_proof_set_delete where txHash is not NULL - var proofSetDeletes []ProofSetDelete +func processPendingDataSetDeletes(ctx context.Context, db *harmonydb.DB) error { + // Query for pdp_data_set_delete where txHash is not NULL + var dataSetDeletes []DataSetDelete - err := db.Select(ctx, &proofSetDeletes, ` + err := db.Select(ctx, &dataSetDeletes, ` SELECT id, set_id, tx_hash - FROM pdp_proof_set_delete + FROM pdp_data_set_delete WHERE tx_hash IS NOT NULL`) if err != nil { - return xerrors.Errorf("failed to select proof set deletes: %w", err) + return xerrors.Errorf("failed to select data set deletes: %w", err) } - if len(proofSetDeletes) == 0 { - // No pending proof set creates + if len(dataSetDeletes) == 0 { + // No pending data set creates return nil } - // Process each proof set delete - for _, psd := range proofSetDeletes { - err := processProofSetDelete(ctx, db, psd) + // Process each data set delete + for _, psd := range dataSetDeletes { + err := processDataSetDelete(ctx, db, psd) if err != nil { - log.Errorf("Failed to process proof set delete for tx %s: %s", psd.DeleteMessageHash, err) + log.Errorf("Failed to process data set delete for tx %s: %s", psd.DeleteMessageHash, err) continue } } @@ -62,7 +62,7 @@ func processPendingProofSetDeletes(ctx context.Context, db *harmonydb.DB) error return nil } -func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDelete) error { +func processDataSetDelete(ctx context.Context, db *harmonydb.DB, psd DataSetDelete) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool @@ -100,9 +100,9 @@ func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDe if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_proof_set_delete WHERE id = $1`, psd.ID) + _, err = tx.Exec(`DELETE FROM pdp_data_set_delete WHERE id = $1`, psd.ID) if err != nil { - return false, xerrors.Errorf("failed to delete row from pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to delete row from pdp_data_set_delete: %w", err) } return true, nil }) @@ -117,28 +117,28 @@ func processProofSetDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetDe comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`UPDATE pdp_proof_set SET removed = TRUE, + n, err := tx.Exec(`UPDATE pdp_data_set SET removed = TRUE, remove_deal_id = $1, remove_message_hash = $2 WHERE id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, + _, err = tx.Exec(`UPDATE pdp_dataset_piece SET removed = TRUE, remove_deal_id = $1, remove_message_hash = $2 - WHERE proof_set_id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) + WHERE data_set_id = $3`, psd.ID, psd.DeleteMessageHash, psd.PID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) + return false, xerrors.Errorf("failed to update pdp_dataset_piece: %w", err) } - _, err = tx.Exec(`DELETE FROM pdp_proof_set_delete WHERE id = $1`, psd.ID) + _, err = tx.Exec(`DELETE FROM pdp_data_set_delete WHERE id = $1`, psd.ID) if err != nil { - return false, xerrors.Errorf("failed to delete row from pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to delete row from pdp_data_set_delete: %w", err) } n, err = tx.Exec(`UPDATE market_mk20_deal diff --git a/tasks/pdp/proofset_addroot_watch.go b/tasks/pdp/dataset_add_piece_watch.go similarity index 62% rename from tasks/pdp/proofset_addroot_watch.go rename to tasks/pdp/dataset_add_piece_watch.go index d21e213a0..5685743a0 100644 --- a/tasks/pdp/proofset_addroot_watch.go +++ b/tasks/pdp/dataset_add_piece_watch.go @@ -19,25 +19,25 @@ import ( ) // Structures to represent database records -type ProofSetRootAdd struct { +type DataSetPieceAdd struct { ID string `db:"id"` Client string `db:"client"` PieceCID2 string `db:"piece_cid_v2"` // pieceCIDV2 PieceCID string `db:"piece_cid"` PieceSize int64 `db:"piece_size"` RawSize int64 `db:"raw_size"` - ProofSet uint64 `db:"proof_set_id"` + DataSet uint64 `db:"data_set_id"` PieceRef int64 `db:"piece_ref"` AddMessageHash string `db:"add_message_hash"` AddMessageIndex int64 `db:"add_message_index"` } -// NewWatcherRootAdd sets up the watcher for proof set root additions -func NewWatcherRootAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { +// NewWatcherPieceAdd sets up the watcher for data set piece additions +func NewWatcherPieceAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetRootAdds(ctx, db) + err := processPendingDataSetPieceAdds(ctx, db) if err != nil { - log.Errorf("Failed to process pending proof set root adds: %s", err) + log.Errorf("Failed to process pending data set piece adds: %s", err) } return nil @@ -46,30 +46,30 @@ func NewWatcherRootAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { } } -// processPendingProofSetRootAdds processes root additions that have been confirmed on-chain -func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB) error { - // Query for pdp_proofset_root_adds entries where add_message_ok = TRUE - var rootAdds []ProofSetRootAdd +// processPendingDataSetPieceAdds processes piece additions that have been confirmed on-chain +func processPendingDataSetPieceAdds(ctx context.Context, db *harmonydb.DB) error { + // Query for pdp_dataset_piece_adds entries where add_message_ok = TRUE + var pieceAdds []DataSetPieceAdd - err := db.Select(ctx, &rootAdds, ` - SELECT id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, piece_ref, add_message_hash, add_message_index + err := db.Select(ctx, &pieceAdds, ` + SELECT id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, piece_ref, add_message_hash, add_message_index FROM pdp_pipeline - WHERE after_add_root = TRUE AND after_add_root_msg = FALSE + WHERE after_add_piece = TRUE AND after_add_piece_msg = FALSE `) if err != nil { - return xerrors.Errorf("failed to select proof set root adds: %w", err) + return xerrors.Errorf("failed to select data set piece adds: %w", err) } - if len(rootAdds) == 0 { + if len(pieceAdds) == 0 { // No pending root adds return nil } - // Process each root addition - for _, rootAdd := range rootAdds { - err := processProofSetRootAdd(ctx, db, rootAdd) + // Process each piece addition + for _, pieceAdd := range pieceAdds { + err := processDataSetPieceAdd(ctx, db, pieceAdd) if err != nil { - log.Errorf("Failed to process root add for tx %s: %s", rootAdd.AddMessageHash, err) + log.Errorf("Failed to process piece add for tx %s: %s", pieceAdd.AddMessageHash, err) continue } } @@ -77,25 +77,25 @@ func processPendingProofSetRootAdds(ctx context.Context, db *harmonydb.DB) error return nil } -func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd ProofSetRootAdd) error { +func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd DataSetPieceAdd) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool err := db.QueryRow(ctx, `SELECT tx_success, tx_receipt FROM message_waits_eth WHERE signed_tx_hash = $1 AND tx_success IS NOT NULL - AND tx_receipt IS NOT NULL`, rootAdd.AddMessageHash).Scan(&txSuccess, &txReceiptJSON) + AND tx_receipt IS NOT NULL`, pieceAdd.AddMessageHash).Scan(&txSuccess, &txReceiptJSON) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", rootAdd.AddMessageHash) + return xerrors.Errorf("tx hash %s is either missing from watch table or is not yet processed by watcher", pieceAdd.AddMessageHash) } - return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", rootAdd.AddMessageHash, err) + return xerrors.Errorf("failed to get tx_receipt for tx %s: %w", pieceAdd.AddMessageHash, err) } // Unmarshal the tx_receipt JSON into types.Receipt var txReceipt types.Receipt err = json.Unmarshal(txReceiptJSON, &txReceipt) if err != nil { - return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", rootAdd.AddMessageHash, err) + return xerrors.Errorf("failed to unmarshal tx_receipt for tx %s: %w", pieceAdd.AddMessageHash, err) } if !txSuccess { @@ -107,14 +107,14 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd Proof jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), '{complete}', to_jsonb(true), true ) - WHERE id = $2;`, "Transaction failed", rootAdd.ID) + WHERE id = $2;`, "Transaction failed", pieceAdd.ID) if err != nil { return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) } if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, rootAdd.ID) + _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, pieceAdd.ID) if err != nil { return false, xerrors.Errorf("failed to clean up pdp pipeline: %w", err) } @@ -136,41 +136,41 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd Proof } // Get the event definition - event, exists := pdpABI.Events["RootsAdded"] + event, exists := pdpABI.Events["PiecesAdded"] if !exists { - return fmt.Errorf("RootsAdded event not found in ABI") + return fmt.Errorf("PiecesAdded event not found in ABI") } - var rootIds []uint64 + var pieceIds []uint64 eventFound := false // Iterate over the logs in the receipt for _, vLog := range txReceipt.Logs { - // Check if the log corresponds to the RootsAdded event + // Check if the log corresponds to the PiecesAdded event if len(vLog.Topics) > 0 && vLog.Topics[0] == event.ID { // The setId is an indexed parameter in Topics[1], but we don't need it here - // as we already have the proofset ID from the database + // as we already have the dataset ID from the database - // Parse the non-indexed parameter (rootIds array) from the data + // Parse the non-indexed parameter (pieceIds array) from the data unpacked, err := event.Inputs.Unpack(vLog.Data) if err != nil { return fmt.Errorf("failed to unpack log data: %w", err) } - // Extract the rootIds array + // Extract the pieceIds array if len(unpacked) == 0 { return fmt.Errorf("no unpacked data found in log") } - // Convert the unpacked rootIds ([]interface{} containing *big.Int) to []uint64 - bigIntRootIds, ok := unpacked[0].([]*big.Int) + // Convert the unpacked pieceIds ([]interface{} containing *big.Int) to []uint64 + bigIntPieceIds, ok := unpacked[0].([]*big.Int) if !ok { return fmt.Errorf("failed to convert unpacked data to array") } - rootIds = make([]uint64, len(bigIntRootIds)) - for i := range bigIntRootIds { - rootIds[i] = bigIntRootIds[i].Uint64() + pieceIds = make([]uint64, len(bigIntPieceIds)) + for i := range bigIntPieceIds { + pieceIds[i] = bigIntPieceIds[i].Uint64() } eventFound = true @@ -180,32 +180,32 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd Proof } if !eventFound { - return fmt.Errorf("RootsAdded event not found in receipt") + return fmt.Errorf("PiecesAdded event not found in receipt") } - rootId := rootIds[rootAdd.AddMessageIndex] + pieceId := pieceIds[pieceAdd.AddMessageIndex] - // Insert into message_waits_eth and pdp_proofset_roots + // Insert into message_waits_eth and pdp_dataset_pieces comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - // Update proof set for initialization upon first add + // Update data set for initialization upon first add _, err = tx.Exec(` - UPDATE pdp_proof_set SET init_ready = true + UPDATE pdp_data_set SET init_ready = true WHERE id = $1 AND prev_challenge_request_epoch IS NULL AND challenge_request_msg_hash IS NULL AND prove_at_epoch IS NULL - `, rootAdd.ProofSet) + `, pieceAdd.DataSet) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } - // Insert into pdp_proofset_roots + // Insert into pdp_dataset_piece n, err := tx.Exec(` - INSERT INTO pdp_proofset_root ( - proof_set_id, + INSERT INTO pdp_dataset_piece ( + data_set_id, client, piece_cid_v2, piece_cid, piece_size, raw_size, - root, + piece, piece_ref, add_deal_id, add_message_hash, @@ -213,26 +213,26 @@ func processProofSetRootAdd(ctx context.Context, db *harmonydb.DB, rootAdd Proof ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) `, - rootAdd.ProofSet, - rootAdd.Client, - rootAdd.PieceCID2, - rootAdd.PieceCID, - rootAdd.PieceSize, - rootAdd.RawSize, - rootId, - rootAdd.PieceRef, - rootAdd.ID, - rootAdd.AddMessageHash, - rootAdd.AddMessageIndex, + pieceAdd.DataSet, + pieceAdd.Client, + pieceAdd.PieceCID2, + pieceAdd.PieceCID, + pieceAdd.PieceSize, + pieceAdd.RawSize, + pieceId, + pieceAdd.PieceRef, + pieceAdd.ID, + pieceAdd.AddMessageHash, + pieceAdd.AddMessageIndex, ) if err != nil { - return false, xerrors.Errorf("failed to insert into pdp_proofset_root: %w", err) + return false, xerrors.Errorf("failed to insert into pdp_dataset_piece: %w", err) } if n != 1 { - return false, xerrors.Errorf("incorrect number of rows inserted for pdp_proofset_root: %d", n) + return false, xerrors.Errorf("incorrect number of rows inserted for pdp_dataset_piece: %d", n) } - n, err = tx.Exec(`UPDATE pdp_pipeline SET after_add_root_msg = TRUE WHERE id = $1`, rootAdd.ID) + n, err = tx.Exec(`UPDATE pdp_pipeline SET after_add_piece_msg = TRUE WHERE id = $1`, pieceAdd.ID) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } diff --git a/tasks/pdp/proofset_delete_root_watch.go b/tasks/pdp/dataset_delete_root_watch.go similarity index 68% rename from tasks/pdp/proofset_delete_root_watch.go rename to tasks/pdp/dataset_delete_root_watch.go index ea8453b9e..7e727f681 100644 --- a/tasks/pdp/proofset_delete_root_watch.go +++ b/tasks/pdp/dataset_delete_root_watch.go @@ -15,18 +15,18 @@ import ( chainTypes "github.com/filecoin-project/lotus/chain/types" ) -type ProofSetRootDelete struct { - ID string `db:"id"` - ProofSet uint64 `db:"set_id"` - Roots []int64 `db:"roots"` - Hash string `db:"tx_hash"` +type DataSetPieceDelete struct { + ID string `db:"id"` + DataSet uint64 `db:"set_id"` + Pieces []int64 `db:"pieces"` + Hash string `db:"tx_hash"` } -func NewWatcherRootDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { +func NewWatcherPieceDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingProofSetRootDeletes(ctx, db) + err := processPendingDataSetPieceDeletes(ctx, db) if err != nil { - log.Errorf("Failed to process pending proof set creates: %s", err) + log.Errorf("Failed to process pending data set creates: %s", err) } return nil }); err != nil { @@ -34,22 +34,22 @@ func NewWatcherRootDelete(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { } } -func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB) error { - var proofSetRootDeletes []ProofSetRootDelete - err := db.Select(ctx, &proofSetRootDeletes, ` - SELECT id, tx_hash, roots, set_id FROM pdp_root_delete WHERE tx_hash IS NOT NULL`) +func processPendingDataSetPieceDeletes(ctx context.Context, db *harmonydb.DB) error { + var dataSetPieceDeletes []DataSetPieceDelete + err := db.Select(ctx, &dataSetPieceDeletes, ` + SELECT id, tx_hash, pieces, set_id FROM pdp_piece_delete WHERE tx_hash IS NOT NULL`) if err != nil { - return xerrors.Errorf("failed to select proof set root deletes: %w", err) + return xerrors.Errorf("failed to select data set piece deletes: %w", err) } - if len(proofSetRootDeletes) == 0 { + if len(dataSetPieceDeletes) == 0 { return nil } - for _, psd := range proofSetRootDeletes { - err := processProofSetRootDelete(ctx, db, psd) + for _, psd := range dataSetPieceDeletes { + err := processDataSetPieceDelete(ctx, db, psd) if err != nil { - log.Errorf("Failed to process proof set root delete for tx %s: %s", psd.Hash, err) + log.Errorf("Failed to process data set piece delete for tx %s: %s", psd.Hash, err) continue } } @@ -57,7 +57,7 @@ func processPendingProofSetRootDeletes(ctx context.Context, db *harmonydb.DB) er return nil } -func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofSetRootDelete) error { +func processDataSetPieceDelete(ctx context.Context, db *harmonydb.DB, psd DataSetPieceDelete) error { var txReceiptJSON []byte var txSuccess bool err := db.QueryRow(ctx, `SELECT tx_receipt, tx_success FROM message_waits_eth WHERE signed_tx_hash = $1 @@ -90,9 +90,9 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_root_delete WHERE id = $1`, psd.ID) + _, err = tx.Exec(`DELETE FROM pdp_piece_delete WHERE id = $1`, psd.ID) if err != nil { - return false, xerrors.Errorf("failed to delete row from pdp_root_delete: %w", err) + return false, xerrors.Errorf("failed to delete row from pdp_piece_delete: %w", err) } return true, nil }) @@ -106,12 +106,12 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS } comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`UPDATE pdp_proofset_root SET removed = TRUE, + n, err := tx.Exec(`UPDATE pdp_dataset_piece SET removed = TRUE, remove_deal_id = $1, remove_message_hash = $2 - WHERE proof_set_id = $3 AND root = ANY($4)`, psd.ID, psd.Hash, psd.ProofSet, psd.Roots) + WHERE data_set_id = $3 AND piece = ANY($4)`, psd.ID, psd.Hash, psd.DataSet, psd.Pieces) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proofset_root: %w", err) + return false, xerrors.Errorf("failed to update pdp_dataset_piece: %w", err) } if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) @@ -125,9 +125,9 @@ func processProofSetRootDelete(ctx context.Context, db *harmonydb.DB, psd ProofS if n != 1 { return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) } - _, err = tx.Exec(`DELETE FROM pdp_root_delete WHERE id = $1`, psd.ID) + _, err = tx.Exec(`DELETE FROM pdp_piece_delete WHERE id = $1`, psd.ID) if err != nil { - return false, xerrors.Errorf("failed to delete row from pdp_root_delete: %w", err) + return false, xerrors.Errorf("failed to delete row from pdp_piece_delete: %w", err) } return true, nil }) diff --git a/tasks/pdp/task_add_proofset.go b/tasks/pdp/task_add_data_set.go similarity index 69% rename from tasks/pdp/task_add_proofset.go rename to tasks/pdp/task_add_data_set.go index 2d228c41f..226841b08 100644 --- a/tasks/pdp/task_add_proofset.go +++ b/tasks/pdp/task_add_data_set.go @@ -2,7 +2,6 @@ package pdp import ( "context" - "database/sql" "errors" "strings" "time" @@ -22,15 +21,15 @@ import ( "github.com/filecoin-project/curio/tasks/message" ) -type PDPTaskAddProofSet struct { +type PDPTaskAddDataSet struct { db *harmonydb.DB sender *message.SenderETH ethClient *ethclient.Client filClient PDPServiceNodeApi } -func NewPDPTaskAddProofSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskAddProofSet { - return &PDPTaskAddProofSet{ +func NewPDPTaskAddDataSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskAddDataSet { + return &PDPTaskAddDataSet{ db: db, sender: sender, ethClient: ethClient, @@ -38,20 +37,20 @@ func NewPDPTaskAddProofSet(db *harmonydb.DB, sender *message.SenderETH, ethClien } } -func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (p *PDPTaskAddDataSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() var pcreates []struct { RecordKeeper string `db:"record_keeper"` ExtraData []byte `db:"extra_data"` } - err = p.db.Select(ctx, &pcreates, `SELECT record_keeper, extra_data FROM pdp_proof_set_create WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + err = p.db.Select(ctx, &pcreates, `SELECT record_keeper, extra_data FROM pdp_data_set_create WHERE task_id = $1 AND tx_hash IS NULL`, taskID) if err != nil { return false, xerrors.Errorf("failed to get task details from DB: %w", err) } if len(pcreates) != 1 { - return false, xerrors.Errorf("incorrect rows for proofset create found for taskID %d", taskID) + return false, xerrors.Errorf("incorrect rows for dataset create found for taskID %d", taskID) } pcreate := pcreates[0] @@ -81,7 +80,7 @@ func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() boo } // Pack the method call data - data, err := abiData.Pack("createProofSet", recordKeeperAddr, extraDataBytes) + data, err := abiData.Pack("createDataSet", recordKeeperAddr, extraDataBytes) if err != nil { return false, xerrors.Errorf("packing data: %w", err) } @@ -97,21 +96,21 @@ func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() boo ) // Send the transaction using SenderETH - reason := "pdp-mkproofset" + reason := "pdp-create-data-set" txHash, err := p.sender.Send(ctx, fromAddress, tx, reason) if err != nil { return false, xerrors.Errorf("sending transaction: %w", err) } - // Insert into message_waits_eth and pdp_proofset_creates + // Insert into message_waits_eth and pdp_data_set_create txHashLower := strings.ToLower(txHash.Hex()) comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`UPDATE pdp_proof_set_create SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + n, err := tx.Exec(`UPDATE pdp_data_set_create SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set_create: %w", err) } if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_create: %d", n) + return false, xerrors.Errorf("incorrect number of rows updated for pdp_data_set_create: %d", n) } _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") if err != nil { @@ -131,14 +130,14 @@ func (p *PDPTaskAddProofSet) Do(taskID harmonytask.TaskID, stillOwned func() boo return true, nil } -func (p *PDPTaskAddProofSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (p *PDPTaskAddDataSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { return &ids[0], nil } -func (p *PDPTaskAddProofSet) TypeDetails() harmonytask.TaskTypeDetails { +func (p *PDPTaskAddDataSet) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "PDPAddProofSet", + Name: "PDPAddDataSet", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -150,27 +149,27 @@ func (p *PDPTaskAddProofSet) TypeDetails() harmonytask.TaskTypeDetails { } } -func (p *PDPTaskAddProofSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { +func (p *PDPTaskAddDataSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { var stop bool for !stop { taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_proof_set_create WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) + err := tx.QueryRow(`SELECT id FROM pdp_data_set_create WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil } - return false, xerrors.Errorf("failed to query pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to query pdp_data_set_create: %w", err) } if did == "" { return false, xerrors.Errorf("no valid id found for taskID") } - _, err = tx.Exec(`UPDATE pdp_proof_set_create SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) + _, err = tx.Exec(`UPDATE pdp_data_set_create SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_create: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set_create: %w", err) } stop = false // we found a task to schedule, keep going @@ -183,12 +182,11 @@ func (p *PDPTaskAddProofSet) schedule(ctx context.Context, taskFunc harmonytask. } // getSenderAddress retrieves the sender address from the database where role = 'pdp' limit 1 -func (p *PDPTaskAddProofSet) getSenderAddress(ctx context.Context) (common.Address, error) { - // TODO: Update this function +func (p *PDPTaskAddDataSet) getSenderAddress(ctx context.Context) (common.Address, error) { var addressStr string err := p.db.QueryRow(ctx, `SELECT address FROM eth_keys WHERE role = 'pdp' LIMIT 1`).Scan(&addressStr) if err != nil { - if errors.Is(err, sql.ErrNoRows) { + if errors.Is(err, pgx.ErrNoRows) { return common.Address{}, errors.New("no sender address with role 'pdp' found") } return common.Address{}, err @@ -197,7 +195,7 @@ func (p *PDPTaskAddProofSet) getSenderAddress(ctx context.Context) (common.Addre return address, nil } -func (p *PDPTaskAddProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} +func (p *PDPTaskAddDataSet) Adder(taskFunc harmonytask.AddTaskFunc) {} -var _ harmonytask.TaskInterface = &PDPTaskAddProofSet{} -var _ = harmonytask.Reg(&PDPTaskAddProofSet{}) +var _ harmonytask.TaskInterface = &PDPTaskAddDataSet{} +var _ = harmonytask.Reg(&PDPTaskAddDataSet{}) diff --git a/tasks/pdp/task_addroot.go b/tasks/pdp/task_add_piece.go similarity index 65% rename from tasks/pdp/task_addroot.go rename to tasks/pdp/task_add_piece.go index dc0f555f8..54370c760 100644 --- a/tasks/pdp/task_addroot.go +++ b/tasks/pdp/task_add_piece.go @@ -30,53 +30,53 @@ type PDPServiceNodeApi interface { ChainHead(ctx context.Context) (*types2.TipSet, error) } -type PDPTaskAddRoot struct { +type PDPTaskAddPiece struct { db *harmonydb.DB sender *message.SenderETH ethClient *ethclient.Client } -func NewPDPTaskAddRoot(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskAddRoot { - return &PDPTaskAddRoot{ +func NewPDPTaskAddPiece(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskAddPiece { + return &PDPTaskAddPiece{ db: db, sender: sender, ethClient: ethClient, } } -func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (p *PDPTaskAddPiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() - var addRoots []struct { - ID string `db:"id"` - PieceCid string `db:"piece_cid"` - PieceCid2 string `db:"piece_cid_v2"` - ProofSetID int64 `db:"proof_set_id"` - ExtraData []byte `db:"extra_data"` - PieceRef string `db:"piece_ref"` + var addPieces []struct { + ID string `db:"id"` + PieceCid string `db:"piece_cid"` + PieceCid2 string `db:"piece_cid_v2"` + DataSetID int64 `db:"data_set_id"` + ExtraData []byte `db:"extra_data"` + PieceRef string `db:"piece_ref"` } - err = p.db.Select(ctx, &addRoots, `SELECT id, piece_cid, piece_cid_v2, proof_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_root_task_id = $1 AND after_add_root = FALSE`, taskID) + err = p.db.Select(ctx, &addPieces, `SELECT id, piece_cid, piece_cid_v2, data_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_piece_task_id = $1 AND after_add_piece = FALSE`, taskID) if err != nil { - return false, xerrors.Errorf("failed to select addRoot: %w", err) + return false, xerrors.Errorf("failed to select add piece: %w", err) } - if len(addRoots) == 0 { - return false, xerrors.Errorf("no addRoot found for taskID %d", taskID) + if len(addPieces) == 0 { + return false, xerrors.Errorf("no add piece found for taskID %d", taskID) } - if len(addRoots) > 1 { - return false, xerrors.Errorf("multiple addRoot found for taskID %d", taskID) + if len(addPieces) > 1 { + return false, xerrors.Errorf("multiple add piece found for taskID %d", taskID) } - addRoot := addRoots[0] + addPiece := addPieces[0] - pcid, err := cid.Parse(addRoot.PieceCid) + pcid, err := cid.Parse(addPiece.PieceCid) if err != nil { return false, xerrors.Errorf("failed to parse piece cid: %w", err) } - pcid2, err := cid.Parse(addRoot.PieceCid2) + pcid2, err := cid.Parse(addPiece.PieceCid2) if err != nil { return false, xerrors.Errorf("failed to parse piece cid: %w", err) } @@ -93,19 +93,19 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) } - rootDataArray := []contract.RootData{ + pieceDataArray := []contract.PieceData{ { - Root: struct{ Data []byte }{Data: pcid.Bytes()}, + Piece: struct{ Data []byte }{Data: pcid.Bytes()}, RawSize: new(big.Int).SetUint64(uint64(pi.Size.Unpadded())), }, } - proofSetID := new(big.Int).SetUint64(uint64(addRoot.ProofSetID)) + dataSetID := new(big.Int).SetUint64(uint64(addPiece.DataSetID)) // Prepare the Ethereum transaction // Pack the method call data // The extraDataBytes variable is now correctly populated above - data, err := abiData.Pack("addRoots", proofSetID, rootDataArray, addRoot.ExtraData) + data, err := abiData.Pack("addPieces", dataSetID, pieceDataArray, addPiece.ExtraData) if err != nil { return false, xerrors.Errorf("packing data: %w", err) } @@ -121,8 +121,8 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return false, xerrors.Errorf("failed to instantiate PDPVerifier contract at %s: %w", pdpVerifierAddress.Hex(), err) } - // Get the sender address for this proofset - owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + // Get the sender address for this dataset + owner, _, err := pdpVerifier.GetDataSetStorageProvider(callOpts, dataSetID) if err != nil { return false, xerrors.Errorf("failed to get owner: %w", err) } @@ -138,7 +138,7 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( ) // Send the transaction using SenderETH - reason := "pdp-addroots" + reason := "pdp-add-piece" txHash, err := p.sender.Send(ctx, owner, txEth, reason) if err != nil { return false, xerrors.Errorf("sending transaction: %w", err) @@ -146,7 +146,7 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( txHashLower := strings.ToLower(txHash.Hex()) - // Insert into message_waits_eth and pdp_proofset_roots + // Insert into message_waits_eth and pdp_dataset_piece _, err = p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Insert into message_waits_eth _, err = tx.Exec(` @@ -158,10 +158,10 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( } n, err := tx.Exec(`UPDATE pdp_pipeline SET - after_add_root = TRUE, - add_root_task_id = NULL, + after_add_piece = TRUE, + add_piece_task_id = NULL, add_message_hash = $2 - WHERE add_root_task_id = $1`, taskID, txHashLower) + WHERE add_piece_task_id = $1`, taskID, txHashLower) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -178,14 +178,14 @@ func (p *PDPTaskAddRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( return true, nil } -func (p *PDPTaskAddRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (p *PDPTaskAddPiece) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { return &ids[0], nil } -func (p *PDPTaskAddRoot) TypeDetails() harmonytask.TaskTypeDetails { +func (p *PDPTaskAddPiece) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "PDPAddRoot", + Name: "PDPAddPiece", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -197,7 +197,7 @@ func (p *PDPTaskAddRoot) TypeDetails() harmonytask.TaskTypeDetails { } } -func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { +func (p *PDPTaskAddPiece) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { var stop bool for !stop { taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { @@ -205,9 +205,9 @@ func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddT var did string err := tx.QueryRow(`SELECT id FROM pdp_pipeline - WHERE add_root_task_id IS NULL - AND after_add_root = FALSE - AND after_add_root_msg = FALSE + WHERE add_piece_task_id IS NULL + AND after_add_piece = FALSE + AND after_add_piece_msg = FALSE AND aggregated = TRUE LIMIT 1`).Scan(&did) if err != nil { @@ -220,7 +220,7 @@ func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddT return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_pipeline SET add_root_task_id = $1 WHERE id = $2 AND after_add_root = FALSE AND after_add_root_msg = FALSE AND aggregated = TRUE`, id, did) + _, err = tx.Exec(`UPDATE pdp_pipeline SET add_piece_task_id = $1 WHERE id = $2 AND after_add_piece = FALSE AND after_add_piece_msg = FALSE AND aggregated = TRUE`, id, did) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -234,7 +234,7 @@ func (p *PDPTaskAddRoot) schedule(ctx context.Context, taskFunc harmonytask.AddT return nil } -func (p *PDPTaskAddRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} +func (p *PDPTaskAddPiece) Adder(taskFunc harmonytask.AddTaskFunc) {} -var _ harmonytask.TaskInterface = &PDPTaskAddRoot{} -var _ = harmonytask.Reg(&PDPTaskAddRoot{}) +var _ harmonytask.TaskInterface = &PDPTaskAddPiece{} +var _ = harmonytask.Reg(&PDPTaskAddPiece{}) diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index 12e3d1356..9c102e3da 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -279,11 +279,11 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b retv := deal.Products.RetrievalV1 n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12, $13)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting aggregated piece in PDP pipeline: %w", err) } diff --git a/tasks/pdp/task_delete_rootset.go b/tasks/pdp/task_delete_data_set.go similarity index 67% rename from tasks/pdp/task_delete_rootset.go rename to tasks/pdp/task_delete_data_set.go index a647c8b88..e0ed526ab 100644 --- a/tasks/pdp/task_delete_rootset.go +++ b/tasks/pdp/task_delete_data_set.go @@ -22,15 +22,15 @@ import ( "github.com/filecoin-project/curio/tasks/message" ) -type PDPTaskDeleteProofSet struct { +type PDPTaskDeleteDataSet struct { db *harmonydb.DB sender *message.SenderETH ethClient *ethclient.Client filClient PDPServiceNodeApi } -func NewPDPTaskDeleteProofSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskDeleteProofSet { - return &PDPTaskDeleteProofSet{ +func NewPDPTaskDeleteDataSet(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client, filClient PDPServiceNodeApi) *PDPTaskDeleteDataSet { + return &PDPTaskDeleteDataSet{ db: db, sender: sender, ethClient: ethClient, @@ -38,27 +38,27 @@ func NewPDPTaskDeleteProofSet(db *harmonydb.DB, sender *message.SenderETH, ethCl } } -func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (p *PDPTaskDeleteDataSet) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() var pdeletes []struct { SetID int64 `db:"set_id"` ExtraData []byte `db:"extra_data"` } - err = p.db.Select(ctx, &pdeletes, `SELECT set_id, extra_data FROM pdp_proof_set_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + err = p.db.Select(ctx, &pdeletes, `SELECT set_id, extra_data FROM pdp_data_set_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) if err != nil { return false, xerrors.Errorf("failed to get task details from DB: %w", err) } if len(pdeletes) != 1 { - return false, xerrors.Errorf("incorrect rows for proofset delete found for taskID %d", taskID) + return false, xerrors.Errorf("incorrect rows for dataset delete found for taskID %d", taskID) } pdelete := pdeletes[0] extraDataBytes := []byte{} - proofSetID := new(big.Int).SetUint64(uint64(pdelete.SetID)) + dataSetID := new(big.Int).SetUint64(uint64(pdelete.SetID)) if pdelete.ExtraData != nil { extraDataBytes = pdelete.ExtraData @@ -76,8 +76,8 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() Context: ctx, } - // Get the sender address for this proofset - owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + // Get the sender address for this dataset + owner, _, err := pdpVerifier.GetDataSetStorageProvider(callOpts, dataSetID) if err != nil { return false, xerrors.Errorf("failed to get owner: %w", err) } @@ -90,7 +90,7 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() } // Pack the method call data - data, err := abiData.Pack("deleteProofSet", proofSetID, extraDataBytes) + data, err := abiData.Pack("deleteDataSet", dataSetID, extraDataBytes) if err != nil { return false, xerrors.Errorf("packing data: %w", err) } @@ -106,22 +106,22 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() ) // Send the transaction using SenderETH - reason := "pdp-rmproofset" + reason := "pdp-delete-data-set" txHash, err := p.sender.Send(ctx, owner, tx, reason) if err != nil { return false, xerrors.Errorf("sending transaction: %w", err) } - // Insert into message_waits_eth and pdp_proof_set_delete + // Insert into message_waits_eth and pdp_data_set_delete txHashLower := strings.ToLower(txHash.Hex()) comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`UPDATE pdp_proof_set_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + n, err := tx.Exec(`UPDATE pdp_data_set_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set_delete: %w", err) } if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_proof_set_delete: %d", n) + return false, xerrors.Errorf("incorrect number of rows updated for pdp_data_set_delete: %d", n) } _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") @@ -142,14 +142,14 @@ func (p *PDPTaskDeleteProofSet) Do(taskID harmonytask.TaskID, stillOwned func() return true, nil } -func (p *PDPTaskDeleteProofSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (p *PDPTaskDeleteDataSet) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { return &ids[0], nil } -func (p *PDPTaskDeleteProofSet) TypeDetails() harmonytask.TaskTypeDetails { +func (p *PDPTaskDeleteDataSet) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "PDPDelProofSet", + Name: "PDPDelDataSet", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -161,27 +161,27 @@ func (p *PDPTaskDeleteProofSet) TypeDetails() harmonytask.TaskTypeDetails { } } -func (p *PDPTaskDeleteProofSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { +func (p *PDPTaskDeleteDataSet) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { var stop bool for !stop { taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_proof_set_delete WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) + err := tx.QueryRow(`SELECT id FROM pdp_data_set_delete WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil } - return false, xerrors.Errorf("failed to query pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to query pdp_data_set_delete: %w", err) } if did == "" { return false, xerrors.Errorf("no valid id found for taskID") } - _, err = tx.Exec(`UPDATE pdp_proof_set_delete SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) + _, err = tx.Exec(`UPDATE pdp_data_set_delete SET task_id = $1 WHERE id = $2 AND tx_hash IS NULL`, id, did) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set_delete: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set_delete: %w", err) } stop = false // we found a task to schedule, keep going @@ -193,7 +193,7 @@ func (p *PDPTaskDeleteProofSet) schedule(ctx context.Context, taskFunc harmonyta return nil } -func (p *PDPTaskDeleteProofSet) Adder(taskFunc harmonytask.AddTaskFunc) {} +func (p *PDPTaskDeleteDataSet) Adder(taskFunc harmonytask.AddTaskFunc) {} -var _ harmonytask.TaskInterface = &PDPTaskDeleteProofSet{} -var _ = harmonytask.Reg(&PDPTaskDeleteProofSet{}) +var _ harmonytask.TaskInterface = &PDPTaskDeleteDataSet{} +var _ = harmonytask.Reg(&PDPTaskDeleteDataSet{}) diff --git a/tasks/pdp/task_delete_root.go b/tasks/pdp/task_delete_piece.go similarity index 63% rename from tasks/pdp/task_delete_root.go rename to tasks/pdp/task_delete_piece.go index 3e3c69cb2..e4ed7805c 100644 --- a/tasks/pdp/task_delete_root.go +++ b/tasks/pdp/task_delete_piece.go @@ -22,29 +22,29 @@ import ( "github.com/filecoin-project/curio/tasks/message" ) -type PDPTaskDeleteRoot struct { +type PDPTaskDeletePiece struct { db *harmonydb.DB sender *message.SenderETH ethClient *ethclient.Client } -func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (p *PDPTaskDeletePiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() var rdeletes []struct { ID string `db:"id"` SetID int64 `db:"set_id"` - Roots []int64 `db:"roots"` + Pieces []int64 `db:"pieces"` ExtraData []byte `db:"extra_data"` } - err = p.db.Select(ctx, &rdeletes, `SELECT id, set_id, roots, extra_data FROM pdp_root_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) + err = p.db.Select(ctx, &rdeletes, `SELECT id, set_id, pieces, extra_data FROM pdp_piece_delete WHERE task_id = $1 AND tx_hash IS NULL`, taskID) if err != nil { return false, xerrors.Errorf("failed to get task details from DB: %w", err) } if len(rdeletes) != 1 { - return false, xerrors.Errorf("incorrect rows for delete root found for taskID %d", taskID) + return false, xerrors.Errorf("incorrect rows for delete piece found for taskID %d", taskID) } rdelete := rdeletes[0] @@ -55,7 +55,7 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool extraDataBytes = rdelete.ExtraData } - proofSetID := new(big.Int).SetUint64(uint64(rdelete.SetID)) + dataSetID := new(big.Int).SetUint64(uint64(rdelete.SetID)) pdpContracts := contract.ContractAddresses() pdpVerifierAddress := pdpContracts.PDPVerifier @@ -69,15 +69,15 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool Context: ctx, } - // Get the sender address for this proofset - owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, proofSetID) + // Get the sender address for this dataset + owner, _, err := pdpVerifier.GetDataSetStorageProvider(callOpts, dataSetID) if err != nil { return false, xerrors.Errorf("failed to get owner: %w", err) } - var roots []*big.Int - for _, root := range rdelete.Roots { - roots = append(roots, new(big.Int).SetUint64(uint64(root))) + var pieces []*big.Int + for _, piece := range rdelete.Pieces { + pieces = append(pieces, new(big.Int).SetUint64(uint64(piece))) } abiData, err := contract.PDPVerifierMetaData.GetAbi() @@ -85,15 +85,8 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) } - for i := range roots { - log.Errorf("root: %d", roots[i].Uint64()) - } - log.Errorf("roots: %v", roots) - log.Errorf("proofSetID: %d", proofSetID.Uint64()) - log.Errorf("extraDataBytes: %s", extraDataBytes) - // Pack the method call data - data, err := abiData.Pack("scheduleRemovals", proofSetID, roots, extraDataBytes) + data, err := abiData.Pack("schedulePieceDeletions", dataSetID, pieces, extraDataBytes) if err != nil { return false, xerrors.Errorf("packing data: %w", err) } @@ -109,22 +102,22 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool ) // Send the transaction using SenderETH - reason := "pdp-rmroot" + reason := "pdp-remove-piece" txHash, err := p.sender.Send(ctx, owner, tx, reason) if err != nil { return false, xerrors.Errorf("sending transaction: %w", err) } - // Insert into message_waits_eth and pdp_proof_set_delete + // Insert into message_waits_eth and pdp_data_set_delete txHashLower := strings.ToLower(txHash.Hex()) comm, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - n, err := tx.Exec(`UPDATE pdp_root_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) + n, err := tx.Exec(`UPDATE pdp_piece_delete SET tx_hash = $1, task_id = NULL WHERE task_id = $2`, txHashLower, taskID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_root_delete: %w", err) + return false, xerrors.Errorf("failed to update pdp_piece_delete: %w", err) } if n != 1 { - return false, xerrors.Errorf("incorrect number of rows updated for pdp_root_delete: %d", n) + return false, xerrors.Errorf("incorrect number of rows updated for pdp_piece_delete: %d", n) } _, err = tx.Exec(`INSERT INTO message_waits_eth (signed_tx_hash, tx_status) VALUES ($1, $2)`, txHashLower, "pending") @@ -132,6 +125,9 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool return false, xerrors.Errorf("failed to insert into message_waits_eth: %w", err) } return true, nil + + // TODO: INSERT IPNI and Index removal tasks + }, harmonydb.OptionRetry()) if err != nil { @@ -145,14 +141,14 @@ func (p *PDPTaskDeleteRoot) Do(taskID harmonytask.TaskID, stillOwned func() bool return true, nil } -func (p *PDPTaskDeleteRoot) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (p *PDPTaskDeletePiece) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { return &ids[0], nil } -func (p *PDPTaskDeleteRoot) TypeDetails() harmonytask.TaskTypeDetails { +func (p *PDPTaskDeletePiece) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "PDPDeleteRoot", + Name: "PDPDeletePiece", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -164,29 +160,29 @@ func (p *PDPTaskDeleteRoot) TypeDetails() harmonytask.TaskTypeDetails { } } -func (p *PDPTaskDeleteRoot) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { +func (p *PDPTaskDeletePiece) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { var stop bool for !stop { taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule var did string - err := tx.QueryRow(`SELECT id FROM pdp_root_delete + err := tx.QueryRow(`SELECT id FROM pdp_piece_delete WHERE task_id IS NULL AND tx_hash IS NULL LIMIT 1`).Scan(&did) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil } - return false, xerrors.Errorf("failed to query pdp_root_delete: %w", err) + return false, xerrors.Errorf("failed to query pdp_piece_delete: %w", err) } if did == "" { return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_root_delete SET task_id = $1 WHERE id = $2 AND task_id IS NULL AND tx_hash IS NULL`, id, did) + _, err = tx.Exec(`UPDATE pdp_piece_delete SET task_id = $1 WHERE id = $2 AND task_id IS NULL AND tx_hash IS NULL`, id, did) if err != nil { - return false, xerrors.Errorf("failed to update pdp_root_delete: %w", err) + return false, xerrors.Errorf("failed to update pdp_piece_delete: %w", err) } stop = false // we found a task to schedule, keep going @@ -198,15 +194,15 @@ func (p *PDPTaskDeleteRoot) schedule(ctx context.Context, taskFunc harmonytask.A return nil } -func (p *PDPTaskDeleteRoot) Adder(taskFunc harmonytask.AddTaskFunc) {} +func (p *PDPTaskDeletePiece) Adder(taskFunc harmonytask.AddTaskFunc) {} -func NewPDPTaskDeleteRoot(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskDeleteRoot { - return &PDPTaskDeleteRoot{ +func NewPDPTaskDeletePiece(db *harmonydb.DB, sender *message.SenderETH, ethClient *ethclient.Client) *PDPTaskDeletePiece { + return &PDPTaskDeletePiece{ db: db, sender: sender, ethClient: ethClient, } } -var _ harmonytask.TaskInterface = &PDPTaskDeleteRoot{} -var _ = harmonytask.Reg(&PDPTaskDeleteRoot{}) +var _ harmonytask.TaskInterface = &PDPTaskDeletePiece{} +var _ = harmonytask.Reg(&PDPTaskDeletePiece{}) diff --git a/tasks/pdp/task_init_pp.go b/tasks/pdp/task_init_pp.go index 8b637b900..d85cb1ba8 100644 --- a/tasks/pdp/task_init_pp.go +++ b/tasks/pdp/task_init_pp.go @@ -2,12 +2,13 @@ package pdp import ( "context" - "database/sql" + "errors" "math/big" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -48,31 +49,31 @@ func NewInitProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil return nil } - // Now query the db for proof sets needing nextProvingPeriod inital call + // Now query the db for data sets needing nextProvingPeriod initial call var toCallInit []struct { - ProofSetID int64 `db:"id"` + DataSetID int64 `db:"id"` } err := db.Select(ctx, &toCallInit, ` SELECT id - FROM pdp_proof_set + FROM pdp_data_set WHERE challenge_request_task_id IS NULL AND init_ready AND prove_at_epoch IS NULL `) - if err != nil && err != sql.ErrNoRows { - return xerrors.Errorf("failed to select proof sets needing nextProvingPeriod: %w", err) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("failed to select data sets needing nextProvingPeriod: %w", err) } for _, ps := range toCallInit { ipp.addFunc.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { - // Update pdp_proof_set to set challenge_request_task_id = id + // Update pdp_data_set to set challenge_request_task_id = id affected, err := tx.Exec(` - UPDATE pdp_proof_set + UPDATE pdp_data_set SET challenge_request_task_id = $1 WHERE id = $2 AND challenge_request_task_id IS NULL - `, id, ps.ProofSetID) + `, id, ps.DataSetID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_set: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if affected == 0 { // Someone else might have already scheduled the task @@ -92,31 +93,42 @@ func NewInitProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() - // Select the proof set where challenge_request_task_id = taskID - var proofSetID int64 + // Select the data set where challenge_request_task_id = taskID + var dataSetID int64 err = ipp.db.QueryRow(ctx, ` SELECT id - FROM pdp_proof_set + FROM pdp_data_set WHERE challenge_request_task_id = $1 - `, taskID).Scan(&proofSetID) - if err == sql.ErrNoRows { - // No matching proof set, task is done (something weird happened, and e.g another task was spawned in place of this one) + `, taskID).Scan(&dataSetID) + if errors.Is(err, pgx.ErrNoRows) { + // No matching data set, task is done (something weird happened, and e.g. another task was spawned in place of this one) return true, nil } if err != nil { - return false, xerrors.Errorf("failed to query pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to query pdp_data_set: %w", err) } - // Get the listener address for this proof set from the PDPVerifier contract + // Get the listener address for this data set from the PDPVerifier contract pdpVerifier, err := contract.NewPDPVerifier(contract.ContractAddresses().PDPVerifier, ipp.ethClient) if err != nil { return false, xerrors.Errorf("failed to instantiate PDPVerifier contract: %w", err) } - listenerAddr, err := pdpVerifier.GetProofSetListener(nil, big.NewInt(proofSetID)) + // Check if the data set has any leaves (pieces) before attempting to initialize proving period + leafCount, err := pdpVerifier.GetDataSetLeafCount(nil, big.NewInt(dataSetID)) if err != nil { - return false, xerrors.Errorf("failed to get listener address for proof set %d: %w", proofSetID, err) + return false, xerrors.Errorf("failed to get leaf count for data set %d: %w", dataSetID, err) + } + if leafCount.Cmp(big.NewInt(0)) == 0 { + // No leaves in the data set yet, skip initialization + // Return done=false to retry later (the task will be retried by the scheduler) + return false, xerrors.Errorf("no leaves in data set %d, skipping initialization", dataSetID) + } + + listenerAddr, err := pdpVerifier.GetDataSetListener(nil, big.NewInt(dataSetID)) + if err != nil { + return false, xerrors.Errorf("failed to get listener address for data set %d: %w", dataSetID, err) } // Determine the next challenge window start by consulting the listener @@ -146,7 +158,7 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( return false, xerrors.Errorf("failed to get PDPVerifier ABI: %w", err) } - data, err := abiData.Pack("nextProvingPeriod", big.NewInt(proofSetID), init_prove_at, []byte{}) + data, err := abiData.Pack("nextProvingPeriod", big.NewInt(dataSetID), init_prove_at, []byte{}) if err != nil { return false, xerrors.Errorf("failed to pack data: %w", err) } @@ -166,7 +178,7 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( return false, nil } - fromAddress, _, err := pdpVerifier.GetProofSetOwner(nil, big.NewInt(proofSetID)) + fromAddress, _, err := pdpVerifier.GetDataSetStorageProvider(nil, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get default sender address: %w", err) } @@ -186,19 +198,19 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( // Update the database in a transaction _, err = ipp.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - // Update pdp_proof_sets + // Update pdp_data_set affected, err := tx.Exec(` - UPDATE pdp_proof_set + UPDATE pdp_data_set SET challenge_request_msg_hash = $1, prev_challenge_request_epoch = $2, prove_at_epoch = $3 WHERE id = $4 - `, txHash.Hex(), ts.Height(), init_prove_at.Uint64(), proofSetID) + `, txHash.Hex(), ts.Height(), init_prove_at.Uint64(), data) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if affected == 0 { - return false, xerrors.Errorf("pdp_proof_sets update affected 0 rows") + return false, xerrors.Errorf("pdp_data_set update affected 0 rows") } // Insert into message_waits_eth diff --git a/tasks/pdp/task_next_pp.go b/tasks/pdp/task_next_pp.go index 29cde60a5..e3f0d6c8a 100644 --- a/tasks/pdp/task_next_pp.go +++ b/tasks/pdp/task_next_pp.go @@ -2,11 +2,12 @@ package pdp import ( "context" - "database/sql" + "errors" "math/big" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -47,31 +48,31 @@ func NewNextProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil return nil } - // Now query the db for proof sets needing nextProvingPeriod + // Now query the db for data sets needing nextProvingPeriod var toCallNext []struct { - ProofSetID int64 `db:"id"` + DataSetID int64 `db:"id"` } err := db.Select(ctx, &toCallNext, ` SELECT id - FROM pdp_proof_set + FROM pdp_data_set WHERE challenge_request_task_id IS NULL AND (prove_at_epoch + challenge_window) <= $1 `, apply.Height()) - if err != nil && err != sql.ErrNoRows { - return xerrors.Errorf("failed to select proof sets needing nextProvingPeriod: %w", err) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return xerrors.Errorf("failed to select data sets needing nextProvingPeriod: %w", err) } for _, ps := range toCallNext { n.addFunc.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { - // Update pdp_proof_sets to set challenge_request_task_id = id + // Update pdp_data_set to set challenge_request_task_id = id affected, err := tx.Exec(` - UPDATE pdp_proof_set + UPDATE pdp_data_set SET challenge_request_task_id = $1 WHERE id = $2 AND challenge_request_task_id IS NULL - `, id, ps.ProofSetID) + `, id, ps.DataSetID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if affected == 0 { // Someone else might have already scheduled the task @@ -90,31 +91,31 @@ func NewNextProvingPeriodTask(db *harmonydb.DB, ethClient *ethclient.Client, fil func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() - // Select the proof set where challenge_request_task_id = taskID - var proofSetID int64 + // Select the data set where challenge_request_task_id = taskID + var dataSetID int64 err = n.db.QueryRow(ctx, ` SELECT id - FROM pdp_proof_set + FROM pdp_data_set WHERE challenge_request_task_id = $1 AND prove_at_epoch IS NOT NULL - `, taskID).Scan(&proofSetID) - if err == sql.ErrNoRows { - // No matching proof set, task is done (something weird happened, and e.g another task was spawned in place of this one) + `, taskID).Scan(&dataSetID) + if errors.Is(err, pgx.ErrNoRows) { + // No matching data set, task is done (something weird happened, and e.g another task was spawned in place of this one) return true, nil } if err != nil { - return false, xerrors.Errorf("failed to query pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to query pdp_data_set: %w", err) } - // Get the listener address for this proof set from the PDPVerifier contract + // Get the listener address for this data set from the PDPVerifier contract pdpVerifier, err := contract.NewPDPVerifier(contract.ContractAddresses().PDPVerifier, n.ethClient) if err != nil { return false, xerrors.Errorf("failed to instantiate PDPVerifier contract: %w", err) } - listenerAddr, err := pdpVerifier.GetProofSetListener(nil, big.NewInt(proofSetID)) + listenerAddr, err := pdpVerifier.GetDataSetListener(nil, big.NewInt(dataSetID)) if err != nil { - return false, xerrors.Errorf("failed to get listener address for proof set %d: %w", proofSetID, err) + return false, xerrors.Errorf("failed to get listener address for data set %d: %w", dataSetID, err) } // Determine the next challenge window start by consulting the listener @@ -122,7 +123,7 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() if err != nil { return false, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) } - next_prove_at, err := provingSchedule.NextChallengeWindowStart(nil, big.NewInt(proofSetID)) + next_prove_at, err := provingSchedule.NextChallengeWindowStart(nil, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get next challenge window start: %w", err) } @@ -137,7 +138,7 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() return false, xerrors.Errorf("failed to get PDPVerifier ABI: %w", err) } - data, err := abiData.Pack("nextProvingPeriod", big.NewInt(proofSetID), next_prove_at, []byte{}) + data, err := abiData.Pack("nextProvingPeriod", big.NewInt(dataSetID), next_prove_at, []byte{}) if err != nil { return false, xerrors.Errorf("failed to pack data: %w", err) } @@ -157,7 +158,7 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() return false, nil } - fromAddress, _, err := pdpVerifier.GetProofSetOwner(nil, big.NewInt(proofSetID)) + fromAddress, _, err := pdpVerifier.GetDataSetStorageProvider(nil, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get default sender address: %w", err) } @@ -177,19 +178,19 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() // Update the database in a transaction _, err = n.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - // Update pdp_proof_sets + // Update pdp_data_set affected, err := tx.Exec(` - UPDATE pdp_proof_set + UPDATE pdp_data_set SET challenge_request_msg_hash = $1, prev_challenge_request_epoch = $2, prove_at_epoch = $3 WHERE id = $4 - `, txHash.Hex(), ts.Height(), next_prove_at.Uint64(), proofSetID) + `, txHash.Hex(), ts.Height(), next_prove_at.Uint64(), dataSetID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if affected == 0 { - return false, xerrors.Errorf("pdp_proof_sets update affected 0 rows") + return false, xerrors.Errorf("pdp_data_set update affected 0 rows") } // Insert into message_waits_eth diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index 2d125beba..7dd05ad3f 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -2,7 +2,6 @@ package pdp import ( "context" - "database/sql" "encoding/binary" "encoding/hex" "errors" @@ -18,6 +17,7 @@ import ( "github.com/ipfs/go-cid" "github.com/minio/sha256-simd" "github.com/samber/lo" + "github.com/yugabyte/pgx/v5" "golang.org/x/crypto/sha3" "golang.org/x/xerrors" @@ -71,7 +71,7 @@ func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethC idx: idx, } - // ProveTasks are created on pdp_proof_sets entries where + // ProveTasks are created on pdp_data_set entries where // challenge_request_msg_hash is not null (=not yet landed) err := chainSched.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { @@ -86,13 +86,13 @@ func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethC pt.addFunc.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { // Select proof sets ready for proving - var proofSets []struct { + var dataSets []struct { ID int64 `db:"id"` } - err := tx.Select(&proofSets, ` + err := tx.Select(&dataSets, ` SELECT p.id - FROM pdp_proof_set p + FROM pdp_data_set p INNER JOIN message_waits_eth mw on mw.signed_tx_hash = p.challenge_request_msg_hash WHERE p.challenge_request_msg_hash IS NOT NULL AND mw.tx_success = TRUE AND p.prove_at_epoch < $1 LIMIT 2 @@ -101,37 +101,37 @@ func NewProveTask(chainSched *chainsched.CurioChainSched, db *harmonydb.DB, ethC return false, xerrors.Errorf("failed to select proof sets: %w", err) } - if len(proofSets) == 0 { + if len(dataSets) == 0 { // No proof sets to process return false, nil } // Determine if there might be more proof sets to process - more = len(proofSets) > 1 + more = len(dataSets) > 1 // Process the first proof set - todo := proofSets[0] + todo := dataSets[0] - // Insert a new task into pdp_prove_tasks + // Insert a new task into pdp_proving_tasks affected, err := tx.Exec(` - INSERT INTO pdp_prove_tasks (proofset, task_id) + INSERT INTO pdp_proving_tasks (data_set_id, task_id) VALUES ($1, $2) ON CONFLICT DO NOTHING `, todo.ID, id) if err != nil { - return false, xerrors.Errorf("failed to insert into pdp_prove_tasks: %w", err) + return false, xerrors.Errorf("failed to insert into pdp_proving_tasks: %w", err) } if affected == 0 { return false, nil } - // Update pdp_proof_sets to set next_challenge_possible = FALSE + // Update pdp_data_set to set next_challenge_possible = FALSE affected, err = tx.Exec(` - UPDATE pdp_proof_set + UPDATE pdp_data_set SET challenge_request_msg_hash = NULL WHERE id = $1 AND challenge_request_msg_hash IS NOT NULL `, todo.ID) if err != nil { - return false, xerrors.Errorf("failed to update pdp_proof_sets: %w", err) + return false, xerrors.Errorf("failed to update pdp_data_set: %w", err) } if affected == 0 { more = false @@ -160,13 +160,13 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done ctx := context.Background() // Retrieve proof set and challenge epoch for the task - var proofSetID int64 + var dataSetID int64 err = p.db.QueryRow(context.Background(), ` - SELECT proofset - FROM pdp_prove_tasks + SELECT data_set_id + FROM pdp_proving_tasks WHERE task_id = $1 - `, taskID).Scan(&proofSetID) + `, taskID).Scan(&dataSetID) if err != nil { return false, xerrors.Errorf("failed to get task details: %w", err) } @@ -184,7 +184,7 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done } // Proof parameters - challengeEpoch, err := pdpVerifier.GetNextChallengeEpoch(callOpts, big.NewInt(proofSetID)) + challengeEpoch, err := pdpVerifier.GetNextChallengeEpoch(callOpts, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get next challenge epoch: %w", err) } @@ -194,7 +194,7 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("failed to get chain randomness from beacon for pdp prove: %w", err) } - proofs, err := p.GenerateProofs(ctx, pdpVerifier, proofSetID, seed, contract.NumChallenges) + proofs, err := p.GenerateProofs(ctx, pdpVerifier, dataSetID, seed, contract.NumChallenges) if err != nil { return false, xerrors.Errorf("failed to generate proofs: %w", err) } @@ -204,7 +204,7 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("failed to get PDPVerifier ABI: %w", err) } - data, err := abiData.Pack("provePossession", big.NewInt(proofSetID), proofs) + data, err := abiData.Pack("provePossession", big.NewInt(dataSetID), proofs) if err != nil { return false, xerrors.Errorf("failed to pack data: %w", err) } @@ -227,12 +227,12 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done proofStr += "] ] ]" - log.Infof("PDP Prove Task: proofSetID: %d, taskID: %d, proofs: %s", proofSetID, taskID, proofStr) + log.Infof("PDP Prove Task: dataSetID: %d, taskID: %d, proofs: %s", dataSetID, taskID, proofStr) } */ // If gas used is 0 fee is maximized gasFee := big.NewInt(0) - proofFee, err := pdpVerifier.CalculateProofFee(callOpts, big.NewInt(proofSetID), gasFee) + proofFee, err := pdpVerifier.CalculateProofFee(callOpts, big.NewInt(dataSetID), gasFee) if err != nil { return false, xerrors.Errorf("failed to calculate proof fee: %w", err) } @@ -240,8 +240,8 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done // Add 2x buffer for certainty proofFee = new(big.Int).Mul(proofFee, big.NewInt(3)) - // Get the sender address for this proofset - owner, _, err := pdpVerifier.GetProofSetOwner(callOpts, big.NewInt(proofSetID)) + // Get the sender address for this dataset + owner, _, err := pdpVerifier.GetDataSetStorageProvider(callOpts, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get owner: %w", err) } @@ -262,7 +262,7 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done ) log.Infow("PDP Prove Task", - "proofSetID", proofSetID, + "dataSetID", dataSetID, "taskID", taskID, "proofs", proofs, "data", hex.EncodeToString(data), @@ -283,46 +283,40 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("failed to send transaction: %w", err) } - // Remove the roots previously scheduled for deletion - err = p.cleanupDeletedRoots(ctx, proofSetID, pdpVerifier) - if err != nil { - return false, xerrors.Errorf("failed to cleanup deleted roots: %w", err) - } - - log.Infow("PDP Prove Task: transaction sent", "txHash", txHash, "proofSetID", proofSetID, "taskID", taskID) + log.Infow("PDP Prove Task: transaction sent", "txHash", txHash, "dataSetID", dataSetID, "taskID", taskID) // Task completed successfully return true, nil } -func (p *ProveTask) GenerateProofs(ctx context.Context, pdpService *contract.PDPVerifier, proofSetID int64, seed abi.Randomness, numChallenges int) ([]contract.PDPVerifierProof, error) { +func (p *ProveTask) GenerateProofs(ctx context.Context, pdpService *contract.PDPVerifier, dataSetID int64, seed abi.Randomness, numChallenges int) ([]contract.PDPVerifierProof, error) { proofs := make([]contract.PDPVerifierProof, numChallenges) callOpts := &bind.CallOpts{ Context: ctx, } - totalLeafCount, err := pdpService.GetChallengeRange(callOpts, big.NewInt(proofSetID)) + totalLeafCount, err := pdpService.GetChallengeRange(callOpts, big.NewInt(dataSetID)) if err != nil { return nil, xerrors.Errorf("failed to get proof set leaf count: %w", err) } totalLeaves := totalLeafCount.Uint64() challenges := lo.Times(numChallenges, func(i int) int64 { - return generateChallengeIndex(seed, proofSetID, i, totalLeaves) + return generateChallengeIndex(seed, dataSetID, i, totalLeaves) }) - rootId, err := pdpService.FindRootIds(callOpts, big.NewInt(proofSetID), lo.Map(challenges, func(i int64, _ int) *big.Int { return big.NewInt(i) })) + pieceId, err := pdpService.FindPieceIds(callOpts, big.NewInt(dataSetID), lo.Map(challenges, func(i int64, _ int) *big.Int { return big.NewInt(i) })) if err != nil { return nil, xerrors.Errorf("failed to find root IDs: %w", err) } for i := 0; i < numChallenges; i++ { - root := rootId[i] + piece := pieceId[i] - proof, err := p.proveRoot(ctx, proofSetID, root.RootId.Int64(), root.Offset.Int64()) + proof, err := p.proveRoot(ctx, dataSetID, piece.PieceId.Int64(), piece.Offset.Int64()) if err != nil { - return nil, xerrors.Errorf("failed to prove root %d (%d, %d, %d): %w", i, proofSetID, root.RootId.Int64(), root.Offset.Int64(), err) + return nil, xerrors.Errorf("failed to prove root %d (%d, %d, %d): %w", i, dataSetID, piece.PieceId.Int64(), piece.Offset.Int64(), err) } proofs[i] = proof @@ -331,7 +325,7 @@ func (p *ProveTask) GenerateProofs(ctx context.Context, pdpService *contract.PDP return proofs, nil } -func generateChallengeIndex(seed abi.Randomness, proofSetID int64, proofIndex int, totalLeaves uint64) int64 { +func generateChallengeIndex(seed abi.Randomness, dataSetID int64, proofIndex int, totalLeaves uint64) int64 { // Create a buffer to hold the concatenated data (96 bytes: 32 bytes * 3) data := make([]byte, 0, 96) @@ -339,10 +333,10 @@ func generateChallengeIndex(seed abi.Randomness, proofSetID int64, proofIndex in data = append(data, seed...) - // Convert proofSetID to 32-byte big-endian representation - proofSetIDBigInt := big.NewInt(proofSetID) - proofSetIDBytes := padTo32Bytes(proofSetIDBigInt.Bytes()) - data = append(data, proofSetIDBytes...) + // Convert dataSetID to 32-byte big-endian representation + dataSetIDBigInt := big.NewInt(dataSetID) + dataSetIDBytes := padTo32Bytes(dataSetIDBigInt.Bytes()) + data = append(data, dataSetIDBytes...) // Convert proofIndex to 8-byte big-endian representation proofIndexBytes := make([]byte, 8) @@ -364,7 +358,7 @@ func generateChallengeIndex(seed abi.Randomness, proofSetID int64, proofIndex in // Log for debugging log.Debugw("generateChallengeIndex", "seed", seed, - "proofSetID", proofSetID, + "dataSetID", dataSetID, "proofIndex", proofIndex, "totalLeaves", totalLeaves, "data", hex.EncodeToString(data), @@ -384,14 +378,14 @@ func padTo32Bytes(b []byte) []byte { return padded } -func (p *ProveTask) proveRoot(ctx context.Context, proofSetID int64, rootId int64, challengedLeaf int64) (contract.PDPVerifierProof, error) { +func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64, challengedLeaf int64) (contract.PDPVerifierProof, error) { //const arity = 2 rootChallengeOffset := challengedLeaf * LeafSize var pieceCid string - err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_proofset_root WHERE proof_set_id = $1 AND root_id = $2`, proofSetID, rootId).Scan(&pieceCid) + err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_dataset_piece WHERE data_set_id = $1 AND root_id = $2`, dataSetID, rootId).Scan(&pieceCid) if err != nil { return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get root and subroot: %w", err) } @@ -559,7 +553,7 @@ func (p *ProveTask) getSenderAddress(ctx context.Context, match common.Address) var addressStr string err := p.db.QueryRow(ctx, `SELECT address FROM eth_keys WHERE role = 'pdp' AND address = $1 LIMIT 1`, match.Hex()).Scan(&addressStr) if err != nil { - if errors.Is(err, sql.ErrNoRows) { + if errors.Is(err, pgx.ErrNoRows) { return common.Address{}, errors.New("no sender address with role 'pdp' found") } return common.Address{}, err @@ -568,65 +562,6 @@ func (p *ProveTask) getSenderAddress(ctx context.Context, match common.Address) return address, nil } -func (p *ProveTask) cleanupDeletedRoots(ctx context.Context, proofSetID int64, pdpVerifier *contract.PDPVerifier) error { - - removals, err := pdpVerifier.GetScheduledRemovals(nil, big.NewInt(proofSetID)) - if err != nil { - return xerrors.Errorf("failed to get scheduled removals: %w", err) - } - - // Execute cleanup in a transaction - ok, err := p.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { - - for _, removeID := range removals { - log.Debugw("cleanupDeletedRoots", "removeID", removeID) - // Get the pdp_pieceref ID for the root before deleting - var pdpPieceRefID int64 - err := tx.QueryRow(` - SELECT pdp_pieceref - FROM pdp_proofset_roots - WHERE proofset = $1 AND root_id = $2 - `, proofSetID, removeID.Int64()).Scan(&pdpPieceRefID) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - // Root already deleted, skip - continue - } - return false, xerrors.Errorf("failed to get piece ref for root %d: %w", removeID, err) - } - - // Delete the parked piece ref, this will cascade to the pdp piece ref too - _, err = tx.Exec(` - DELETE FROM parked_piece_refs - WHERE ref_id = $1 - `, pdpPieceRefID) - if err != nil { - return false, xerrors.Errorf("failed to delete parked piece ref %d: %w", pdpPieceRefID, err) - } - - // Delete the root entry - _, err = tx.Exec(` - DELETE FROM pdp_proofset_roots - WHERE proofset = $1 AND root_id = $2 - `, proofSetID, removeID) - if err != nil { - return false, xerrors.Errorf("failed to delete root %d: %w", removeID, err) - } - } - - return true, nil - }, harmonydb.OptionRetry()) - - if err != nil { - return xerrors.Errorf("failed to cleanup deleted roots: %w", err) - } - if !ok { - return xerrors.Errorf("database delete not committed") - } - - return nil -} - func (p *ProveTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { if len(ids) == 0 { return nil, nil diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go index 2100ad1fc..e4fbf7d2c 100644 --- a/tasks/pdp/task_save_cache.go +++ b/tasks/pdp/task_save_cache.go @@ -31,32 +31,32 @@ import ( const MinSizeForCache = uint64(100 * 1024 * 1024) const CacheReadSize = int64(4 * 1024 * 1024) -type TaskSavePDPCache struct { +type TaskPDPSaveCache struct { db *harmonydb.DB cpr *cachedreader.CachedPieceReader idx *indexstore.IndexStore } -func NewTaskSavePDPCache(db *harmonydb.DB, cpr *cachedreader.CachedPieceReader, idx *indexstore.IndexStore) *TaskSavePDPCache { - return &TaskSavePDPCache{ +func NewTaskPDPSaveCache(db *harmonydb.DB, cpr *cachedreader.CachedPieceReader, idx *indexstore.IndexStore) *TaskPDPSaveCache { + return &TaskPDPSaveCache{ db: db, cpr: cpr, idx: idx, } } -func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { +func (t *TaskPDPSaveCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { ctx := context.Background() var saveCaches []struct { - ID string `db:"id"` - PieceCid string `db:"piece_cid_v2"` - ProofSetID int64 `db:"proof_set_id"` - PieceRef string `db:"piece_ref"` + ID string `db:"id"` + PieceCid string `db:"piece_cid_v2"` + DataSetID int64 `db:"data_set_id"` + PieceRef string `db:"piece_ref"` } - err = t.db.Select(ctx, &saveCaches, `SELECT id, piece_cid_v2, proof_set_id, piece_ref FROM pdp_pipeline WHERE save_cache_task_id = $1 AND after_save_cache = FALSE`, taskID) + err = t.db.Select(ctx, &saveCaches, `SELECT id, piece_cid_v2, data_set_id, piece_ref FROM pdp_pipeline WHERE save_cache_task_id = $1 AND after_save_cache = FALSE`, taskID) if err != nil { - return false, xerrors.Errorf("failed to select addRoot: %w", err) + return false, xerrors.Errorf("failed to select rows from pipeline: %w", err) } if len(saveCaches) == 0 { @@ -142,14 +142,14 @@ func (t *TaskSavePDPCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) return true, nil } -func (t *TaskSavePDPCache) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { +func (t *TaskPDPSaveCache) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { return &ids[0], nil } -func (t *TaskSavePDPCache) TypeDetails() harmonytask.TaskTypeDetails { +func (t *TaskPDPSaveCache) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ Max: taskhelp.Max(50), - Name: "SavePDPCache", + Name: "PDPSaveCache", Cost: resources.Resources{ Cpu: 1, Ram: 64 << 20, @@ -161,7 +161,7 @@ func (t *TaskSavePDPCache) TypeDetails() harmonytask.TaskTypeDetails { } } -func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { +func (t *TaskPDPSaveCache) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { var stop bool for !stop { taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { @@ -171,7 +171,7 @@ func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.Ad err := tx.QueryRow(`SELECT id FROM pdp_pipeline WHERE save_cache_task_id IS NULL AND after_save_cache = FALSE - AND after_add_root_msg = TRUE`).Scan(&did) + AND after_add_piece_msg = TRUE`).Scan(&did) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil @@ -182,7 +182,7 @@ func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.Ad return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1 WHERE id = $2 AND after_save_cache = FALSE AND after_add_root_msg = TRUE`, id, did) + _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1 WHERE id = $2 AND after_save_cache = FALSE AND after_add_piece_msg = TRUE`, id, did) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -196,10 +196,10 @@ func (t *TaskSavePDPCache) schedule(ctx context.Context, taskFunc harmonytask.Ad return nil } -func (t *TaskSavePDPCache) Adder(taskFunc harmonytask.AddTaskFunc) {} +func (t *TaskPDPSaveCache) Adder(taskFunc harmonytask.AddTaskFunc) {} -var _ harmonytask.TaskInterface = &TaskSavePDPCache{} -var _ = harmonytask.Reg(&TaskSavePDPCache{}) +var _ harmonytask.TaskInterface = &TaskPDPSaveCache{} +var _ = harmonytask.Reg(&TaskPDPSaveCache{}) // All the code below is a copy+paste of https://github.com/filecoin-project/go-fil-commp-hashhash/blob/master/commp.go // with modification to output the nodes at a specific height diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index a2311df3b..75a43f1a1 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -288,21 +288,6 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) } - _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) - if err != nil { - return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) - } - - _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) - if err != nil { - return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) - } - - _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) - if err != nil { - return false, xerrors.Errorf("deleting parked piece refs: %w", err) - } - refIDUsed = true } } @@ -328,11 +313,11 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } @@ -341,11 +326,11 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } } else { n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, proof_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.ProofSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePayload) + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) } @@ -357,6 +342,21 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } } + _, err = tx.Exec(`DELETE FROM market_mk20_pipeline_waiting WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal from mk20 pipeline waiting: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_deal_chunk WHERE id = $1`, id.String()) + if err != nil { + return false, xerrors.Errorf("deleting deal chunks from mk20 deal: %w", err) + } + + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = ANY($1)`, refIds) + if err != nil { + return false, xerrors.Errorf("deleting parked piece refs: %w", err) + } + return true, nil }, harmonydb.OptionRetry()) diff --git a/tasks/seal/task_submit_commit.go b/tasks/seal/task_submit_commit.go index ad6517442..fccc02db2 100644 --- a/tasks/seal/task_submit_commit.go +++ b/tasks/seal/task_submit_commit.go @@ -33,7 +33,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" - "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/storage/ctladdr" ) @@ -408,15 +407,7 @@ func (s *SubmitCommitTask) createCommitMessage(ctx context.Context, maddr addres } aggParams.SectorProofs = nil // can't be set when aggregating - aggFeeRaw, err := policy.AggregateProveCommitNetworkFee(nv, len(infos), ts.MinTicketBlock().ParentBaseFee) - if err != nil { - return nil, xerrors.Errorf("getting aggregate commit network fee: %s", err) - } - - aggFee := big.Div(big.Mul(aggFeeRaw, big.NewInt(110)), big.NewInt(110)) - - aggCollateral := big.Add(collateral, aggFee) - aggCollateral = s.calculateCollateral(balance, aggCollateral) + aggCollateral := s.calculateCollateral(balance, collateral) goodFunds := big.Add(maxFee, aggCollateral) aggEnc := new(bytes.Buffer) if err := aggParams.MarshalCBOR(aggEnc); err != nil { @@ -427,7 +418,7 @@ func (s *SubmitCommitTask) createCommitMessage(ctx context.Context, maddr addres return nil, err } aggGas := big.Mul(big.Add(ts.MinTicketBlock().ParentBaseFee, aggMsg.GasPremium), big.NewInt(aggMsg.GasLimit)) - aggCost = big.Add(aggGas, aggFee) + aggCost = aggGas } { diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index cb4118ef2..0edad6879 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/commcidv2" "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -87,7 +88,7 @@ func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { } func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { - ticker := time.NewTicker(10 * time.Second) + ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() for { select { @@ -95,6 +96,7 @@ func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { return case <-ticker.C: d.insertDDODealInPipeline(ctx) + d.insertDealInPipelineForUpload(ctx) } } } @@ -159,6 +161,158 @@ func (d *CurioStorageDealMarket) insertDDODealInPipeline(ctx context.Context) { } } +// insertDealInPipelineForUpload start processing deals which are +// 1. Waiting for data +// 2. DataSource defined +// 3. We already have the piece +// We process both DDO and PDP deal in same function +func (d *CurioStorageDealMarket) insertDealInPipelineForUpload(ctx context.Context) { + var deals []struct { + DealID string `db:"id"` + } + err := d.db.Select(ctx, &deals, `SELECT id from market_mk20_upload_waiting WHERE chunked IS NULL AND ref_id IS NULL`) + if err != nil { + log.Errorf("querying mk20 pipeline waiting upload: %s", err) + return + } + + var dealIDs []ulid.ULID + for _, deal := range deals { + id, err := ulid.Parse(deal.DealID) + if err != nil { + log.Errorf("parsing deal id: %s", err) + return + } + dealIDs = append(dealIDs, id) + } + if len(dealIDs) == 0 { + return + } + + for _, id := range dealIDs { + _, err = d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + deal, err := mk20.DealFromTX(tx, id) + if err != nil { + return false, xerrors.Errorf("getting deal from db: %w", err) + } + + if deal.Data == nil { + return false, nil + } + + pi, err := deal.PieceInfo() + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + + var pieceID int64 + // Check if already have the piece and save the user trouble to upload + err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2`, pi.PieceCIDV1.String(), pi.Size).Scan(&pieceID) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // We don't have the piece, let user upload + return false, nil + } else { + // Some other error occurred during select + return false, xerrors.Errorf("checking existing parked piece: %w", err) + } + } + + retv := deal.Products.RetrievalV1 + data := deal.Data + + aggregation := 0 + if data.Format.Aggregate != nil { + aggregation = int(data.Format.Aggregate.Type) + } + + spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) + if err != nil { + return false, fmt.Errorf("getting provider ID: %w", err) + } + + var comm bool + + // Insert DDO deal if present + if deal.Products.DDOV1 != nil { + ddo := deal.Products.DDOV1 + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + // If we have the piece then create reference and insert in pipeline + var pieceRefID int64 + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id`, pieceID, "/PUT").Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", pieceRefID), + } + + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid_v2, piece_cid, piece_size, raw_size, url, + offline, indexing, announce, allocation_id, duration, + piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE, TRUE)`, + id, spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), + false, retv.Indexing, retv.AnnouncePayload, allocationID, ddo.Duration, + 0, aggregation) + if err != nil { + return false, xerrors.Errorf("inserting piece in mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting piece in mk20 pipeline: %d rows affected", n) + } + + comm = true + } + + if deal.Products.PDPV1 != nil { + pdp := deal.Products.PDPV1 + + // If we have the piece then create reference and insert in pipeline + var pieceRefID int64 + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id`, pieceID, "/PUT").Scan(&pieceRefID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12, $13)`, + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) + if err != nil { + return false, xerrors.Errorf("inserting piece in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting piece in PDP pipeline: %d rows affected", n) + } + comm = true + } + + return comm, nil + }) + if err != nil { + log.Errorf("inserting upload deal in pipeline: %s", err) + continue + } + } +} + func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20.Deal) error { spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) if err != nil { @@ -973,3 +1127,119 @@ func (d *CurioStorageDealMarket) processMK20DealIngestion(ctx context.Context) { } } } + +func (d *CurioStorageDealMarket) migratePieceCIDV2(ctx context.Context) { + ticker := time.NewTicker(1 * time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + d.migratePcid(ctx) + } + } +} + +func (d *CurioStorageDealMarket) migratePcid(ctx context.Context) { + // Migrate ipni_chunks table + var pieceCIDs []struct { + PieceCID string `db:"piece_cid"` + } + err := d.db.Select(ctx, &pieceCIDs, `SELECT piece_cid FROM ipni_chunks`) + if err != nil { + log.Errorf("failed to get piece CIDs: %w", err) + return + } + + for _, pieceCID := range pieceCIDs { + pcid, err := cid.Parse(pieceCID.PieceCID) + if err != nil { + log.Errorf("failed to parse piece CID: %w", err) + continue + } + isPcid2 := commcidv2.IsPieceCidV2(pcid) + if isPcid2 { + continue + } + + comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // Check that table market_piece_metadata has a single entry for this piece cid + var count int + err = tx.QueryRow(`SELECT COUNT(*) AS count FROM market_piece_metadata WHERE piece_cid = $1`, pieceCID.PieceCID).Scan(&count) + if err != nil { + return false, xerrors.Errorf("failed to get piece metadata: %w", err) + } + if count != 1 { + return false, xerrors.Errorf("expected to find a single piece metadata entry for piece cid %s", pieceCID.PieceCID) + } + // Get raw size from market_piece_deal table for this piece CID + var rawSize uint64 + err = tx.QueryRow(`SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1`, pieceCID.PieceCID).Scan(&rawSize) + if err != nil { + log.Errorf("failed to get piece deal: %w", err) + } + + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, rawSize) + if err != nil { + return false, xerrors.Errorf("failed to convert to piece cid v2: %w", err) + } + + // Update ipni_chunks table with correct entry + _, err = tx.Exec(`UPDATE ipni_chunks SET piece_cid = $1 WHERE piece_cid = $2`, pcid2.String(), pieceCID.PieceCID) + if err != nil { + return false, xerrors.Errorf("failed to update ipni_chunks table: %w", err) + } + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + log.Errorf("failed to commit transaction: %s", err) + continue + } + if comm { + log.Debugw("piece CID migrated successfully", "piece CID", pieceCID.PieceCID) + } else { + log.Debugw("piece CID not migrated", "piece CID", pieceCID.PieceCID) + } + } + + // Add PieceCIDv2 to ipni table + var pieceInfos []struct { + PieceCID string `db:"piece_cid"` + Size int64 `db:"size"` + RawSize int64 `db:"raw_size"` + } + err = d.db.Select(ctx, &pieceInfos, `SELECT + i.piece_cid, + i.piece_size, + mpd.raw_size + FROM ipni AS i + JOIN LATERAL ( + SELECT d.raw_size + FROM market_piece_deal AS d + WHERE d.piece_cid = i.piece_cid + AND d.piece_length = i.piece_size + LIMIT 1 + ) AS mpd ON true + WHERE i.piece_cid_v2 IS NULL;`) + if err != nil { + log.Errorf("failed to get piece infos: %w", err) + return + } + for _, pieceInfo := range pieceInfos { + pcid, err := cid.Parse(pieceInfo.PieceCID) + if err != nil { + log.Errorf("failed to parse piece CID: %w", err) + } + + pcid2, err := commcidv2.PieceCidV2FromV1(pcid, uint64(pieceInfo.RawSize)) + if err != nil { + log.Errorf("failed to convert to piece cid v2: %w", err) + } + + _, err = d.db.Exec(ctx, `UPDATE ipni SET piece_cid_v2 = $1 WHERE piece_cid = $2 AND piece_size = $3`, pcid2.String(), pieceInfo.PieceCID, pieceInfo.Size) + if err != nil { + log.Errorf("failed to update ipni table: %w", err) + } + } +} diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index f296c7dc5..e6cf13d2a 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -209,6 +209,7 @@ func (d *CurioStorageDealMarket) runPoller(ctx context.Context) { if module == mk20Str { if len(miners) > 0 { go d.pipelineInsertLoop(ctx) + go d.migratePieceCIDV2(ctx) } } } diff --git a/web/api/webrpc/pdp.go b/web/api/webrpc/pdp.go index 697b94158..6bb9aa51c 100644 --- a/web/api/webrpc/pdp.go +++ b/web/api/webrpc/pdp.go @@ -164,12 +164,14 @@ func (a *WebRPC) ImportPDPKey(ctx context.Context, hexPrivateKey string) (string // Insert into the database within a transaction _, err = a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { // Check if the owner_address already exists - var existingAddress string - err := tx.QueryRow(`SELECT address FROM eth_keys WHERE address = $1 AND role = 'pdp'`, address).Scan(&existingAddress) - if err == nil { + var existingAddress bool + + err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM eth_keys WHERE role = 'pdp')`).Scan(&existingAddress) + if err != nil { + return false, xerrors.Errorf("failed to check existing owner address: %v", err) + } + if existingAddress { return false, fmt.Errorf("owner address %s already exists", address) - } else if err != pgx.ErrNoRows { - return false, fmt.Errorf("failed to check existing owner address: %v", err) } // Insert the new owner address and private key From 29f099e2291eaa842b04dbc713aebbc47dec4884 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 13 Aug 2025 17:44:06 +0400 Subject: [PATCH 25/55] lotus version --- .github/image/Dockerfile | 2 +- Makefile | 2 +- market/mk20/types.go | 4 +--- tasks/storage-market/mk20.go | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/image/Dockerfile b/.github/image/Dockerfile index b516e243c..c2afa6de9 100644 --- a/.github/image/Dockerfile +++ b/.github/image/Dockerfile @@ -23,7 +23,7 @@ RUN git submodule update --init RUN go mod download # Stage 2: Install Lotus binary -FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.33.0-devnet AS lotus-test +FROM ghcr.io/filecoin-shipyard/lotus-containers:lotus-v1.33.1-devnet AS lotus-test # Stage 3: Build the final image FROM myoung34/github-runner AS curio-github-runner diff --git a/Makefile b/Makefile index 0b662a0d4..14a8bf740 100644 --- a/Makefile +++ b/Makefile @@ -283,7 +283,7 @@ build_lotus?=0 curio_docker_user?=curio curio_base_image=$(curio_docker_user)/curio-all-in-one:latest-debug ffi_from_source?=0 -lotus_version?=v1.33.0 +lotus_version?=v1.33.1 ifeq ($(build_lotus),1) # v1: building lotus image with provided lotus version diff --git a/market/mk20/types.go b/market/mk20/types.go index 48304f8cf..a80209fb9 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -57,10 +57,8 @@ type DataSource struct { // SourceOffline defines the data source for offline pieces, including raw size information. SourceOffline *DataSourceOffline `json:"source_offline,omitempty"` - // SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import + // SourceHttpPut allow clients to push piece data after deal is accepted SourceHttpPut *DataSourceHttpPut `json:"source_httpput,omitempty"` - - // SourceStorageProvider -> sp IDs/ipni, pieceCids } // PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 0edad6879..b50486508 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -88,7 +88,7 @@ func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { } func (d *CurioStorageDealMarket) pipelineInsertLoop(ctx context.Context) { - ticker := time.NewTicker(30 * time.Second) + ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { select { From 25d75e9d9ae07961c539a9b069a5a8326d34da03 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Fri, 15 Aug 2025 15:30:55 +0400 Subject: [PATCH 26/55] CommPV2 integration with PDP contracts --- cmd/sptool/toolbox_deal_client.go | 11 +- .../harmonydb/sql/20250505-market_mk20.sql | 28 +- itests/pdp_prove_test.go | 2 +- market/ipni/types/types.go | 42 +- market/ipni/types/types_cbor_gen.go | 150 ----- market/mk20/http/http.go | 59 +- market/mk20/mk20.go | 5 +- market/mk20/mk20_upload.go | 450 ++++++++++--- market/mk20/mk20_utils.go | 10 +- market/mk20/pdp_v1.go | 4 +- market/mk20/types.go | 3 + market/mk20/utils.go | 32 + pdp/contract/IPDPProvingSchedule.json | 2 +- pdp/contract/PDPVerifier.abi | 527 +++++++-------- pdp/contract/PDPVerifier.json | 2 +- pdp/contract/addresses.go | 2 +- pdp/contract/pdp_verifier.go | 605 +++++++++++------- pdp/contract/types.go | 5 +- tasks/indexing/task_check_indexes.go | 3 +- tasks/indexing/task_indexing.go | 84 +-- tasks/indexing/task_ipni.go | 15 +- tasks/indexing/task_pdp_indexing.go | 13 +- tasks/indexing/task_pdp_ipni.go | 88 +-- tasks/pdp/dataset_delete_root_watch.go | 2 +- tasks/pdp/task_add_piece.go | 22 +- tasks/pdp/task_aggregation.go | 11 +- tasks/pdp/task_prove.go | 67 +- tasks/piece/task_aggregate_chunks.go | 47 +- tasks/storage-market/storage_market.go | 106 ++- tasks/storage-market/task_aggregation.go | 8 +- web/api/webrpc/market_20.go | 4 +- 31 files changed, 1405 insertions(+), 1004 deletions(-) delete mode 100644 market/ipni/types/types_cbor_gen.go diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 320df54c2..9a885a581 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -2142,6 +2142,7 @@ var mk20ClientChunkUploadCmd = &cli.Command{ purl := hurls[0] log.Debugw("using first URL", "url", purl) tu := mk20.StartUpload{ + RawSize: uint64(size), ChunkSize: chunkSize, } b, err := json.Marshal(tu) @@ -2241,7 +2242,7 @@ var mk20ClientChunkUploadCmd = &cli.Command{ log.Infow("upload complete") //Finalize the upload - resp, err = http.Post(purl.String()+"/market/mk20/uploads/finalize/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) + resp, err = http.Post(purl.String()+"/market/mk20/uploads/finalize/"+dealid.String(), "application/json", nil) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } @@ -2685,6 +2686,8 @@ var mk20ClientUploadCmd = &cli.Command{ return xerrors.Errorf("opening file: %w", err) } + defer f.Close() + stat, err := f.Stat() if err != nil { return xerrors.Errorf("stat file: %w", err) @@ -2695,8 +2698,6 @@ var mk20ClientUploadCmd = &cli.Command{ return xerrors.Errorf("file size is 0") } - f.Close() - api, closer, err := lcli.GetGatewayAPIV1(cctx) if err != nil { return fmt.Errorf("cant setup gateway connection: %w", err) @@ -2750,7 +2751,7 @@ var mk20ClientUploadCmd = &cli.Command{ purl := hurls[0] log.Debugw("using first URL", "url", purl) - req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/upload/"+dealid.String(), f) + req, err := http.NewRequest(http.MethodPut, purl.String()+"/market/mk20/upload/"+dealid.String(), io.NewSectionReader(f, 0, size)) if err != nil { return xerrors.Errorf("failed to create put request: %w", err) } @@ -2771,7 +2772,7 @@ var mk20ClientUploadCmd = &cli.Command{ log.Infow("upload complete") //Finalize the upload - resp, err = http.Post(purl.String()+"/market/mk20/upload/"+dealid.String(), "application/json", bytes.NewReader([]byte{})) + resp, err = http.Post(purl.String()+"/market/mk20/upload/"+dealid.String(), "application/json", nil) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index 15b22bb33..de6815275 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -324,9 +324,9 @@ BEGIN AND OLD.ready_at IS NULL AND NOT (OLD.chunked IS FALSE AND OLD.ref_id IS NOT NULL) THEN NEW.ready_at := NOW() AT TIME ZONE 'UTC'; -END IF; + END IF; -RETURN NEW; + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -347,15 +347,15 @@ BEGIN SELECT 1 FROM market_mk20_deal_chunk WHERE id = NEW.id AND (complete IS NOT TRUE) ) THEN -UPDATE market_mk20_upload_waiting -SET ready_at = NOW() AT TIME ZONE 'UTC' -WHERE id = NEW.id - AND chunked = true - AND ready_at IS NULL; -END IF; -END IF; - -RETURN NEW; + UPDATE market_mk20_upload_waiting + SET ready_at = NOW() AT TIME ZONE 'UTC' + WHERE id = NEW.id + AND chunked = true + AND ready_at IS NULL; + END IF; + END IF; + + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -465,12 +465,12 @@ CREATE TABLE pdp_data_set ( -- Set to true after first root add init_ready BOOLEAN NOT NULL DEFAULT FALSE, - create_deal_id TEXT NOT NULL, -- mk20 deal ID for creating this proofset + create_deal_id TEXT NOT NULL, -- mk20 deal ID for creating this data_set create_message_hash TEXT NOT NULL, removed BOOLEAN DEFAULT FALSE, - remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this proofset + remove_deal_id TEXT DEFAULT NULL, -- mk20 deal ID for removing this data_set remove_message_hash TEXT DEFAULT NULL, unique (create_deal_id), @@ -688,7 +688,7 @@ BEGIN -- Insert the new ad into the ipni table with an automatically assigned order_number INSERT INTO ipni (ad_cid, context_id, metadata, is_rm, previous, provider, addresses, signature, entries, piece_cid_v2, piece_cid, piece_size) - VALUES (_ad_cid, _context_id, metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size); + VALUES (_ad_cid, _context_id, _metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size); -- Update the ipni_head table to set the new ad as the head of the chain INSERT INTO ipni_head (provider, head) diff --git a/itests/pdp_prove_test.go b/itests/pdp_prove_test.go index caea13af5..5ae021b8c 100644 --- a/itests/pdp_prove_test.go +++ b/itests/pdp_prove_test.go @@ -150,7 +150,7 @@ func TestPDPProving(t *testing.T) { require.Equal(t, proofs.Root, digest32) rd := proofs.Root - out := contract.PDPVerifierProof{ + out := contract.IPDPTypesProof{ Leaf: subTreeProof.Leaf, Proof: append(subTreeProof.Proof, proofs.Proof...), } diff --git a/market/ipni/types/types.go b/market/ipni/types/types.go index cc28d7e03..7248571e2 100644 --- a/market/ipni/types/types.go +++ b/market/ipni/types/types.go @@ -2,15 +2,49 @@ package types import ( "github.com/ipfs/go-cid" + "golang.org/x/xerrors" ) -//go:generate cbor-gen-for --map-encoding PieceInfo - -// PieceInfo is used to generate the context CIDs for PDP IPNI ads -type PieceInfo struct { +// PdpIpniContext is used to generate the context bytes for PDP IPNI ads +type PdpIpniContext struct { // PieceCID is piece CID V2 PieceCID cid.Cid // Payload determines if the IPNI ad is TransportFilecoinPieceHttp or TransportIpfsGatewayHttp Payload bool } + +// Marshal encodes the PdpIpniContext into a byte slice containing a single byte for Payload and the byte representation of PieceCID. +func (p *PdpIpniContext) Marshal() ([]byte, error) { + pBytes := p.PieceCID.Bytes() + if len(pBytes) > 63 { + return nil, xerrors.Errorf("piece CID byte length exceeds 63") + } + payloadByte := make([]byte, 1) + if p.Payload { + payloadByte[0] = 1 + } else { + payloadByte[0] = 0 + } + return append(payloadByte, pBytes...), nil +} + +// Unmarshal decodes the provided byte slice into the PdpIpniContext struct, validating its length and extracting the PieceCID and Payload values. +func (p *PdpIpniContext) Unmarshal(b []byte) error { + if len(b) > 64 { + return xerrors.Errorf("byte length exceeds 64") + } + if len(b) < 2 { + return xerrors.Errorf("byte length is less than 2") + } + payload := b[0] == 1 + pcid, err := cid.Cast(b[1:]) + if err != nil { + return err + } + + p.PieceCID = pcid + p.Payload = payload + + return nil +} diff --git a/market/ipni/types/types_cbor_gen.go b/market/ipni/types/types_cbor_gen.go deleted file mode 100644 index bb408fe15..000000000 --- a/market/ipni/types/types_cbor_gen.go +++ /dev/null @@ -1,150 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package types - -import ( - "fmt" - "io" - "math" - "sort" - - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = math.E -var _ = sort.Sort - -func (t *PieceInfo) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Payload (bool) (bool) - if len("Payload") > 8192 { - return xerrors.Errorf("Value in field \"Payload\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Payload"))); err != nil { - return err - } - if _, err := cw.WriteString(string("Payload")); err != nil { - return err - } - - if err := cbg.WriteBool(w, t.Payload); err != nil { - return err - } - - // t.PieceCID (cid.Cid) (struct) - if len("PieceCID") > 8192 { - return xerrors.Errorf("Value in field \"PieceCID\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PieceCID"))); err != nil { - return err - } - if _, err := cw.WriteString(string("PieceCID")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.PieceCID); err != nil { - return xerrors.Errorf("failed to write cid field t.PieceCID: %w", err) - } - - return nil -} - -func (t *PieceInfo) UnmarshalCBOR(r io.Reader) (err error) { - *t = PieceInfo{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("PieceInfo: map struct too large (%d)", extra) - } - - n := extra - - nameBuf := make([]byte, 8) - for i := uint64(0); i < n; i++ { - nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 8192) - if err != nil { - return err - } - - if !ok { - // Field doesn't exist on this type, so ignore it - if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { - return err - } - continue - } - - switch string(nameBuf[:nameLen]) { - // t.Payload (bool) (bool) - case "Payload": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Payload = false - case 21: - t.Payload = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.PieceCID (cid.Cid) (struct) - case "PieceCID": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PieceCID: %w", err) - } - - t.PieceCID = c - - } - - default: - // Field doesn't exist on this type, so ignore it - if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { - return err - } - } - } - - return nil -} diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index d4fe1bbe5..59314a08d 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -1,6 +1,7 @@ package http import ( + "bytes" "context" "embed" _ "embed" @@ -579,30 +580,34 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re return } + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "error reading request body", http.StatusBadRequest) + return + } + defer r.Body.Close() + + log.Debugw("received upload finalize proposal", "id", idStr, "body", string(body)) + + if len(bytes.TrimSpace(body)) == 0 { + log.Debugw("no deal provided, using empty deal to finalize upload", "id", idStr) + mdh.dm.MK20Handler.HandleUploadFinalize(id, nil, w) + return + } + ct := r.Header.Get("Content-Type") - // If Content-Type is not set this is does not require updating the deal if len(ct) == 0 { - log.Infow("received finalize upload proposal without content type", "id", id) - mdh.dm.MK20Handler.HandleUploadFinalize(id, nil, w) + http.Error(w, "missing content type", http.StatusBadRequest) return } - var deal mk20.Deal if ct != "application/json" { log.Errorf("invalid content type: %s", ct) http.Error(w, "invalid content type", http.StatusBadRequest) return } - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %s", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - log.Infow("received upload finalize proposal", "body", string(body)) + var deal mk20.Deal err = json.Unmarshal(body, &deal) if err != nil { @@ -661,6 +666,7 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques } defer r.Body.Close() + body, err := io.ReadAll(r.Body) if err != nil { log.Errorf("error reading request body: %s", err) @@ -760,14 +766,23 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h return } - ct := r.Header.Get("Content-Type") - // If Content-Type is not set, it is not required to update the deal - if len(ct) == 0 { - log.Infow("received finalize upload proposal without content type", "id", id) + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, "error reading request body", http.StatusBadRequest) + return + } + defer r.Body.Close() + + log.Debugw("received serial upload finalize proposal", "id", idStr, "body", string(body)) + + if len(bytes.TrimSpace(body)) == 0 { + log.Debugw("no deal provided, using empty deal to finalize upload", "id", idStr) mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, nil, w) return } + ct := r.Header.Get("Content-Type") + var deal mk20.Deal if ct != "application/json" { log.Errorf("invalid content type: %s", ct) @@ -775,14 +790,6 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h return } - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %s", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - err = json.Unmarshal(body, &deal) if err != nil { log.Errorf("error unmarshaling json: %s", err) @@ -790,7 +797,5 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h return } - log.Infow("received serial upload finalize proposal", "body", string(body)) - mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, &deal, w) } diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index ac1b4d80d..8cf75df84 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -190,7 +190,7 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal, tx *harmonydb.Tx) return err } n, err := tx.Exec(`UPDATE market_mk20_deal - SET ddo_v1 = jsonb_set(ddo_v1, '{deal_id}', to_jsonb($1::text)) + SET ddo_v1 = jsonb_set(ddo_v1, '{deal_id}', to_jsonb($1::bigint)) WHERE id = $2;`, id, deal.Identifier.String()) if err != nil { return err @@ -257,6 +257,7 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal, tx *harmonydb.Tx) } func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { + fmt.Println("I HAVE ENTERED DDO SANITY CHECK") if !lo.Contains(m.miners, deal.Products.DDOV1.Provider) { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, @@ -385,6 +386,8 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe } } + fmt.Println("I HAVE EXITED DDO SANITY CHECK") + return nil, nil } diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index 96c2f26a2..d2ebb5c29 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -9,8 +9,13 @@ import ( "io" "math" "net/http" + "net/url" + "runtime" + "runtime/debug" "time" + "github.com/filecoin-project/go-address" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" @@ -243,6 +248,12 @@ func (m *MK20) HandleUploadStart(ctx context.Context, id ulid.ULID, upload Start // @Return UploadCode func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w http.ResponseWriter) { + if m.maxParallelUploads.Load()+1 > int64(m.cfg.Market.StorageMarketConfig.MK20.MaxParallelChunkUploads) { + log.Errorw("max parallel uploads reached", "deal", id, "chunk", chunk, "error", "max parallel uploads reached") + http.Error(w, "too many parallel uploads for provider", int(UploadRateLimit)) + return + } + ctx := context.Background() defer data.Close() @@ -286,15 +297,14 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w return } - defer func() { - m.maxParallelUploads.Add(-1) - }() - log.Debugw("uploading chunk", "deal", id, "chunk", chunk) chunkSize := chunkDetails[0].Size reader := NewTimeoutLimitReader(data, time.Second*5) m.maxParallelUploads.Add(1) + defer func() { + m.maxParallelUploads.Add(-1) + }() // Generate unique tmp pieceCID and Size for parked_pieces tables wr := new(commp.Calc) @@ -495,6 +505,7 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri http.Error(w, "", int(ErrServerInternalError)) return } + newDeal = ddeal } var valid bool @@ -513,6 +524,48 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri return } + if newDeal.Products.DDOV1 != nil { + rej, err := m.sanitizeDDODeal(ctx, newDeal) + if err != nil { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej != nil { + if rej.HTTPCode == 500 { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", rej.Reason) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej.HTTPCode != 500 { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", rej.Reason) + http.Error(w, rej.Reason, int(rej.HTTPCode)) + return + } + } + } + + if newDeal.Products.PDPV1 != nil { + rej, err := m.sanitizePDPDeal(ctx, newDeal) + if err != nil { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej != nil { + if rej.HTTPCode == 500 { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", rej.Reason) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej.HTTPCode != 500 { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", rej.Reason) + http.Error(w, rej.Reason, int(rej.HTTPCode)) + return + } + } + } + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { // Now update the upload status to trigger the correct pipeline n, err := tx.Exec(`UPDATE market_mk20_deal_chunk SET finalize = TRUE where id = $1`, id.String()) @@ -596,6 +649,12 @@ func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []P } func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseWriter) { + if m.maxParallelUploads.Load()+1 > int64(m.cfg.Market.StorageMarketConfig.MK20.MaxParallelChunkUploads) { + log.Errorw("max parallel uploads reached", "deal", id, "error", "max parallel uploads reached") + http.Error(w, "too many parallel uploads for provider", int(UploadRateLimit)) + return + } + ctx := context.Background() var exists bool err := m.DB.QueryRow(ctx, `SELECT EXISTS ( @@ -614,10 +673,13 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW reader := NewTimeoutLimitReader(body, time.Second*5) m.maxParallelUploads.Add(1) + defer func() { + m.maxParallelUploads.Add(-1) + }() // Generate unique tmp pieceCID and Size for parked_pieces tables wr := new(commp.Calc) - trSize, err := wr.Write([]byte(fmt.Sprintf("%s, %s", id.String(), time.Now().String()))) + trs, err := wr.Write([]byte(fmt.Sprintf("%s, %s", id.String(), time.Now().String()))) if err != nil { log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "error", err) http.Error(w, "", int(UploadServerError)) @@ -628,26 +690,61 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW panic(err) } - tpcid := cid.NewCidV1(cid.FilCommitmentUnsealed, digest) + trSize := uint64(trs) + + tpcid, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + log.Errorw("failed to generate tmp pieceCID", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + + deal, err := DealFromDB(ctx, m.DB, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + return + } + + var havePInfo bool + var pinfo *PieceInfo + + if deal.Data != nil { + pi, err := deal.PieceInfo() + if err != nil { + log.Errorw("failed to get piece info from deal", "deal", id, "error", err) + http.Error(w, "", int(UploadServerError)) + } + + tpcid = pi.PieceCIDV1 + tsize = uint64(pi.Size) + trSize = pi.RawSize + havePInfo = true + pinfo = pi + } + var pnum, refID int64 + pieceExists := true // Generate piece park details with tmp pieceCID and Size comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 - AND piece_raw_size = $3`, tpcid.String(), tsize, trSize).Scan(&pnum) + AND piece_raw_size = $3 + AND complete = true`, tpcid.String(), tsize, trSize).Scan(&pnum) if err != nil { if errors.Is(err, pgx.ErrNoRows) { err = tx.QueryRow(` INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term, skip) - VALUES ($1, $2, $3, FALSE, TRUE) + VALUES ($1, $2, $3, TRUE, TRUE) ON CONFLICT (piece_cid, piece_padded_size, long_term, cleanup_task_id) DO NOTHING RETURNING id`, tpcid.String(), tsize, trSize).Scan(&pnum) if err != nil { return false, xerrors.Errorf("inserting new parked piece and getting id: %w", err) } + pieceExists = false } else { return false, xerrors.Errorf("checking existing parked piece: %w", err) } @@ -655,7 +752,7 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW // Add parked_piece_ref err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, FALSE) RETURNING ref_id`, pnum, "/PUT").Scan(&refID) + VALUES ($1, $2, TRUE) RETURNING ref_id`, pnum, "/PUT").Scan(&refID) if err != nil { return false, xerrors.Errorf("inserting parked piece ref: %w", err) } @@ -684,19 +781,26 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW return } - log.Debugw("tmp piece details generated for the chunk", "deal", id) + // If we know the piece details and already have it then let's return early + if pieceExists && havePInfo { + w.WriteHeader(int(UploadOk)) + } + + if !havePInfo { + log.Debugw("tmp piece details generated for the chunk", "deal", id) + } failed := true defer func() { if failed { - _, err = m.DB.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) - if err != nil { - log.Errorw("failed to delete parked piece ref", "deal", id, "error", err) + _, serr := m.DB.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if serr != nil { + log.Errorw("failed to delete parked piece ref", "deal", id, "error", serr) } - _, err = m.DB.Exec(ctx, `UPDATE market_mk20_upload_waiting SET chunked = NULL WHERE id = $1`, id.String()) - if err != nil { - log.Errorw("failed to update upload waiting", "deal", id, "error", err) + _, serr = m.DB.Exec(ctx, `UPDATE market_mk20_upload_waiting SET chunked = NULL WHERE id = $1`, id.String()) + if serr != nil { + log.Errorw("failed to update upload waiting", "deal", id, "error", serr) } } }() @@ -709,37 +813,112 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW return } + if havePInfo { + if rawSize != pinfo.RawSize { + log.Errorw("piece raw size does not match", "deal", id, "supplied", pinfo.RawSize, "written", rawSize, "error", "piece raw size does not match") + http.Error(w, "piece raw size does not match", int(UploadBadRequest)) + return + } + + if !pi.PieceCID.Equals(pinfo.PieceCIDV1) { + log.Errorw("piece CID does not match", "deal", id, "error", "piece CID does not match") + http.Error(w, "piece CID does not match", int(UploadBadRequest)) + return + } + if pi.Size != pinfo.Size { + log.Errorw("piece size does not match", "deal", id, "error", "piece size does not match") + http.Error(w, "piece size does not match", int(UploadBadRequest)) + return + } + } + log.Debugw("piece stored", "deal", id) // Update piece park details with correct values comm, err = m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - var pid int64 - // Check if we already have the piece, if found then verify access and skip rest of the processing - err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCID.String(), pi.Size).Scan(&pid) - if err == nil { - // If piece exists then check if we can access the data - pr, err := m.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if havePInfo { + n, err := tx.Exec(`UPDATE parked_pieces SET complete = true WHERE id = $1`, pnum) if err != nil { - // We should fail here because any subsequent operation which requires access to data will also fail - // till this error is fixed - if !errors.Is(err, storiface.ErrSectorNotFound) { - return false, fmt.Errorf("failed to get piece reader: %w", err) - } + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + } else { + var pid int64 + var complete bool + // Check if we already have the piece, if found then verify access and skip rest of the processing + err = tx.QueryRow(`SELECT id, complete FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCID.String(), pi.Size).Scan(&pid, &complete) + if err == nil { + if complete { + // If piece exists then check if we can access the data + pr, err := m.sc.PieceReader(ctx, storiface.PieceNumber(pid)) + if err != nil { + // We should fail here because any subsequent operation which requires access to data will also fail + // till this error is fixed + if !errors.Is(err, storiface.ErrSectorNotFound) { + return false, fmt.Errorf("failed to get piece reader: %w", err) + } - // If piece does not exist then we update piece park table to work with new tmpID - // Update ref table's reference to tmp id - _, err = tx.Exec(`UPDATE parked_piece_refs SET piece_id = $1 WHERE piece_id = $2`, pnum, pid) - if err != nil { - return false, xerrors.Errorf("updating parked piece ref: %w", err) - } + // If piece does not exist then we update piece park table to work with new tmpID + // Update ref table's reference to tmp id + _, err = tx.Exec(`UPDATE parked_piece_refs SET piece_id = $1 WHERE piece_id = $2`, pnum, pid) + if err != nil { + return false, xerrors.Errorf("updating parked piece ref: %w", err) + } - // Now delete the original piece which has 404 error - _, err = tx.Exec(`DELETE FROM parked_pieces WHERE id = $1`, pid) - if err != nil { - return false, xerrors.Errorf("deleting parked piece: %w", err) - } + // Now delete the original piece which has 404 error + _, err = tx.Exec(`DELETE FROM parked_pieces WHERE id = $1`, pid) + if err != nil { + return false, xerrors.Errorf("deleting parked piece: %w", err) + } - // Update the tmp entry with correct details + // Update the tmp entry with correct details + n, err := tx.Exec(`UPDATE parked_pieces SET + piece_cid = $1, + piece_padded_size = $2, + piece_raw_size = $3, + complete = true + WHERE id = $4`, + pi.PieceCID.String(), pi.Size, rawSize, pnum) + if err != nil { + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + } else { + defer pr.Close() + // Add parked_piece_ref if no errors + var newRefID int64 + err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, FALSE) RETURNING ref_id`, pid, "/PUT").Scan(&newRefID) + if err != nil { + return false, xerrors.Errorf("inserting parked piece ref: %w", err) + } + + // Remove the tmp refs. This will also delete the new tmp parked_pieces entry + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) + if err != nil { + return false, xerrors.Errorf("deleting tmp parked piece ref: %w", err) + } + // Update refID to be used later + refID = newRefID + } + } else { + n, err := tx.Exec(`UPDATE parked_pieces SET complete = true WHERE id = $1`, pid) + if err != nil { + return false, xerrors.Errorf("updating parked piece: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) + } + } + } else { + if !errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("failed to check if piece already exists: %w", err) + } + // If piece does not exist then let's update the tmp one n, err := tx.Exec(`UPDATE parked_pieces SET piece_cid = $1, piece_padded_size = $2, @@ -753,41 +932,6 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW if n != 1 { return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) } - } else { - defer pr.Close() - // Add parked_piece_ref if no errors - var newRefID int64 - err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) - VALUES ($1, $2, FALSE) RETURNING ref_id`, pid, "/PUT").Scan(&newRefID) - if err != nil { - return false, xerrors.Errorf("inserting parked piece ref: %w", err) - } - - // Remove the tmp refs. This will also delete the parked_pieces entry - _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, refID) - if err != nil { - return false, xerrors.Errorf("deleting tmp parked piece ref: %w", err) - } - // Update refID to be used later - refID = newRefID - } - } else { - if !errors.Is(err, pgx.ErrNoRows) { - return false, fmt.Errorf("failed to check if piece already exists: %w", err) - } - // If piece does not exist then let's update the tmp one - n, err := tx.Exec(`UPDATE parked_pieces SET - piece_cid = $1, - piece_padded_size = $2, - piece_raw_size = $3, - complete = true - WHERE id = $4`, - pi.PieceCID.String(), pi.Size, rawSize, pnum) - if err != nil { - return false, xerrors.Errorf("updating parked piece: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) } } @@ -820,6 +964,15 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW } func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { + defer func() { + if r := recover(); r != nil { + trace := make([]byte, 1<<16) + n := runtime.Stack(trace, false) + log.Errorf("panic occurred: %v\n%s", r, trace[:n]) + debug.PrintStack() + } + }() + ctx := context.Background() var exists bool err := m.DB.QueryRow(ctx, `SELECT EXISTS ( @@ -851,15 +1004,15 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } var pcidStr string - var rawSize, refID, pieceSize int64 + var rawSize, refID, pid, pieceSize int64 - err = m.DB.QueryRow(ctx, `SELECT r.ref_id, p.piece_cid, p.piece_padded_size, p.piece_raw_size + err = m.DB.QueryRow(ctx, `SELECT r.ref_id, p.piece_cid, p.piece_padded_size, p.piece_raw_size, p.id FROM market_mk20_upload_waiting u JOIN parked_piece_refs r ON u.ref_id = r.ref_id JOIN parked_pieces p ON r.piece_id = p.id WHERE u.id = $1 AND p.complete = TRUE - AND p.long_term = TRUE;`, id.String()).Scan(&refID, &pcidStr, &pieceSize, &rawSize) + AND p.long_term = TRUE;`, id.String()).Scan(&refID, &pcidStr, &pieceSize, &rawSize, &pid) if err != nil { log.Errorw("failed to get piece details", "deal", id, "error", err) http.Error(w, "", int(ErrServerInternalError)) @@ -917,6 +1070,54 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo return } + fmt.Println("I HAVE REACHED SANITY CHECK") + + if uDeal.Products.DDOV1 != nil { + rej, err := m.sanitizeDDODeal(ctx, uDeal) + if err != nil { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej != nil { + if rej.HTTPCode == 500 { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", rej.Reason) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej.HTTPCode != 500 { + log.Errorw("failed to sanitize DDO deal", "deal", id, "error", rej.Reason) + http.Error(w, rej.Reason, int(rej.HTTPCode)) + return + } + } + } + + fmt.Println("I HAVE FINISHED SANITIZING DDO DEAL") + + if uDeal.Products.PDPV1 != nil { + rej, err := m.sanitizePDPDeal(ctx, uDeal) + if err != nil { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", err) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej != nil { + if rej.HTTPCode == 500 { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", rej.Reason) + http.Error(w, "", int(ErrServerInternalError)) + return + } + if rej.HTTPCode != 500 { + log.Errorw("failed to sanitize PDP deal", "deal", id, "error", rej.Reason) + http.Error(w, rej.Reason, int(rej.HTTPCode)) + return + } + } + } + + fmt.Println("I HAVE FINISHED SANITIZING PDP DEAL") + comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err = tx.Exec(`DELETE FROM market_mk20_upload_waiting WHERE id = $1`, id.String()) if err != nil { @@ -931,22 +1132,91 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } } - pdp := uDeal.Products.PDPV1 + fmt.Println("I HAVE FINISHED UPDATING DEAL") + retv := uDeal.Products.RetrievalV1 + if retv != nil { + fmt.Println("I HAVE RETRIEVAL V1") + } + data := uDeal.Data - // Insert the PDP pipeline - n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, - pdp.ExtraData, refID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) - if err != nil { - return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + aggregation := 0 + if data.Format.Aggregate != nil { + aggregation = int(data.Format.Aggregate.Type) } - if n != 1 { - return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + + var refUsed bool + + if uDeal.Products.DDOV1 != nil { + ddo := uDeal.Products.DDOV1 + spid, err := address.IDFromAddress(ddo.Provider) + if err != nil { + return false, fmt.Errorf("getting provider ID: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", refID), + } + + var allocationID interface{} + if ddo.AllocationId != nil { + allocationID = *ddo.AllocationId + } else { + allocationID = nil + } + + n, err := tx.Exec(`INSERT INTO market_mk20_pipeline ( + id, sp_id, contract, client, piece_cid_v2, piece_cid, + piece_size, raw_size, url, offline, indexing, announce, + allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, + id.String(), spid, ddo.ContractAddress, uDeal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + pi.Size, pi.RawSize, pieceIDUrl.String(), false, retv.Indexing, retv.AnnouncePayload, + allocationID, ddo.Duration, aggregation, aggregation) + + if err != nil { + return false, xerrors.Errorf("inserting mk20 pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting mk20 pipeline: %d rows affected", n) + } + + log.Debugw("mk20 pipeline created", "deal", id) + + refUsed = true + fmt.Println("I HAVE FINISHED CREATING MK20 PIPELINE") } + + if uDeal.Products.PDPV1 != nil { + pdp := uDeal.Products.PDPV1 + // Insert the PDP pipeline + if refUsed { + err = tx.QueryRow(` + INSERT INTO parked_piece_refs (piece_id, data_url, long_term) + VALUES ($1, $2, TRUE) RETURNING ref_id + `, pid, "/PUT").Scan(&refID) + if err != nil { + return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) + } + } + + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, + id.String(), uDeal.Client.String(), uDeal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, refID, aggregation, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) + } + log.Debugw("PDP pipeline created", "deal", id) + fmt.Println("I HAVE FINISHED CREATING PDP PIPELINE") + } + return true, nil }) @@ -977,7 +1247,7 @@ func removeNotFinalizedUploads(ctx context.Context, db *harmonydb.DB) { err := db.Select(ctx, &deals, `SELECT id, chunked, ref_id, ready_at FROM market_mk20_upload_waiting WHERE chunked IS NOT NULL - AND ready_at <= (NOW() AT TIME ZONE 'UTC') - INTERVAL '60 minutes';`) + AND ready_at <= NOW() AT TIME ZONE 'UTC' - INTERVAL '60 minutes';`) if err != nil { log.Errorw("failed to get not finalized uploads", "error", err) } diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index c20c60f2c..f4a788d70 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -26,10 +26,10 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { var pdp_error, ddo_error sql.NullString err := m.DB.QueryRow(ctx, `SELECT - pdp_v1->>'complete' AS pdp_complete, - pdp_v1->>'error' AS pdp_error, - ddo_v1->>'complete' AS ddo_complete, - ddo_v1->>'error' AS ddo_error + (pdp_v1->>'complete')::boolean AS pdp_complete, + (pdp_v1->>'error')::text AS pdp_error, + (ddo_v1->>'complete')::boolean AS ddo_complete, + (ddo_v1->>'error')::text AS ddo_error FROM market_mk20_deal WHERE id = $1;`, id.String()).Scan(&pdp_complete, &pdp_error, &ddo_complete, &ddo_error) if err != nil { @@ -185,7 +185,7 @@ func NewTimeoutLimitReader(r io.Reader, timeout time.Duration) *TimeoutLimitRead } } -const UploadSizeLimit = int64(1 * 1024 * 1024) +const UploadSizeLimit = int64(1 * 1024 * 1024 * 1024) func (t *TimeoutLimitReader) Read(p []byte) (int, error) { deadline := time.Now().Add(t.timeout) diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 8a6666e7c..242035e23 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -45,7 +45,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er } if ok := p.CreateDataSet || p.DeleteDataSet || p.AddPiece || p.DeletePiece; !ok { - return ErrBadProposal, xerrors.Errorf("deal must have one of the following flags set: create_proof_set, delete_proof_set, add_root, delete_root") + return ErrBadProposal, xerrors.Errorf("deal must have one of the following flags set: create_data_set, delete_data_set, add_piece, delete_piece") } var existingAddress bool @@ -121,7 +121,7 @@ func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, er FROM pdp_dataset_piece r JOIN pdp_data_set s ON r.data_set_id = s.id WHERE r.data_set_id = $1 - AND r.root = ANY($2) + AND r.piece = ANY($2) AND r.removed = FALSE AND s.removed = FALSE;`, pid, p.PieceIDs).Scan(&exists) if err != nil { diff --git a/market/mk20/types.go b/market/mk20/types.go index a80209fb9..022c6160e 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -272,4 +272,7 @@ const ( // UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500. UploadServerError UploadCode = 500 + + // UploadRateLimit indicates that the upload operation is being rate-limited, corresponding to the HTTP status code 429. + UploadRateLimit UploadCode = 429 ) diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 21dac6bf9..8c4ae70c8 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -125,6 +125,7 @@ func (d DataSource) Validate(db *harmonydb.DB) (DealCode, error) { return ErrMalformedDataSource, xerrors.Errorf("aggregate type not supported") } + // If client will supply individual pieces if d.SourceAggregate != nil { code, err := IsDataSourceEnabled(db, d.SourceAggregate.Name()) if err != nil { @@ -192,10 +193,41 @@ func (d DataSource) Validate(db *harmonydb.DB) (DealCode, error) { } } } + if len(d.Format.Aggregate.Sub) > 0 { + return ErrMalformedDataSource, xerrors.Errorf("sub pieces cannot be defined when dataSource is aggregate") + } } else { + // If client will supply pre-aggregated piece if len(d.Format.Aggregate.Sub) == 0 { return ErrMalformedDataSource, xerrors.Errorf("no sub pieces defined under aggregate") } + for _, p := range d.Format.Aggregate.Sub { + err := ValidatePieceCID(p.PieceCID) + if err != nil { + return ErrMalformedDataSource, xerrors.Errorf("invalid piece cid") + } + var ifcar, ifraw bool + if p.Format.Car != nil { + ifcar = true + } + + if p.Format.Aggregate != nil { + return ErrMalformedDataSource, xerrors.Errorf("aggregate of aggregate is not supported") + } + + if p.Format.Raw != nil { + ifraw = true + } + if !ifcar && !ifraw { + return ErrMalformedDataSource, xerrors.Errorf("no format defined for sub piece in aggregate") + } + if ifcar && ifraw { + return ErrMalformedDataSource, xerrors.Errorf("multiple formats defined for sub piece in aggregate") + } + if p.SourceAggregate != nil || p.SourceOffline != nil || p.SourceHTTP != nil || p.SourceHttpPut != nil { + return ErrMalformedDataSource, xerrors.Errorf("sub piece of pre-aggregated piece cannot have source defined") + } + } } } diff --git a/pdp/contract/IPDPProvingSchedule.json b/pdp/contract/IPDPProvingSchedule.json index ebc58af53..009d29c85 100644 --- a/pdp/contract/IPDPProvingSchedule.json +++ b/pdp/contract/IPDPProvingSchedule.json @@ -1 +1 @@ -{"abi":[{"type":"function","name":"challengeWindow","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"getChallengesPerProof","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"getMaxProvingPeriod","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"initChallengeWindowStart","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"nextChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"challengeWindow()":"861a1412","getChallengesPerProof()":"47d3dfe7","getMaxProvingPeriod()":"f2f12333","initChallengeWindowStart()":"21918cea","nextChallengeWindowStart(uint256)":"8bf96d28"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"challengeWindow\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengesPerProof\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getMaxProvingPeriod\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"initChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"challengeWindow()\":{\"returns\":{\"_0\":\"Challenge window size in epochs\"}},\"getChallengesPerProof()\":{\"returns\":{\"_0\":\"Number of challenges required per proof\"}},\"getMaxProvingPeriod()\":{\"returns\":{\"_0\":\"Maximum proving period in epochs\"}},\"nextChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the data set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingWindow\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"challengeWindow()\":{\"notice\":\"Returns the number of epochs at the end of a proving period during which proofs can be submitted\"},\"getChallengesPerProof()\":{\"notice\":\"Returns the required number of challenges/merkle inclusion proofs per data set\"},\"getMaxProvingPeriod()\":{\"notice\":\"Returns the number of epochs allowed before challenges must be resampled\"},\"initChallengeWindowStart()\":{\"notice\":\"Value for initializing the challenge window start for any data set assuming proving period starts now\"},\"nextChallengeWindowStart(uint256)\":{\"notice\":\"Calculates the start of the next challenge window for a given data set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x404a211500ef49c7fddaccc98267dbeb18b5c5077ef57d3025337dbf319faa84\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://62053e73723267e564db5a1e7fa0c9b21fb6241c823061c9d414a26fea600af7\",\"dweb:/ipfs/QmWKrJaFBuL5W521YTaLAZoJFqBskAUQi9y4MijxX92G9F\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"pure","type":"function","name":"challengeWindow","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getChallengesPerProof","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getMaxProvingPeriod","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"initChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"challengeWindow()":{"returns":{"_0":"Challenge window size in epochs"}},"getChallengesPerProof()":{"returns":{"_0":"Number of challenges required per proof"}},"getMaxProvingPeriod()":{"returns":{"_0":"Maximum proving period in epochs"}},"nextChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the data set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"challengeWindow()":{"notice":"Returns the number of epochs at the end of a proving period during which proofs can be submitted"},"getChallengesPerProof()":{"notice":"Returns the required number of challenges/merkle inclusion proofs per data set"},"getMaxProvingPeriod()":{"notice":"Returns the number of epochs allowed before challenges must be resampled"},"initChallengeWindowStart()":{"notice":"Value for initializing the challenge window start for any data set assuming proving period starts now"},"nextChallengeWindowStart(uint256)":{"notice":"Calculates the start of the next challenge window for a given data set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x404a211500ef49c7fddaccc98267dbeb18b5c5077ef57d3025337dbf319faa84","urls":["bzz-raw://62053e73723267e564db5a1e7fa0c9b21fb6241c823061c9d414a26fea600af7","dweb:/ipfs/QmWKrJaFBuL5W521YTaLAZoJFqBskAUQi9y4MijxX92G9F"],"license":"Apache-2.0 OR MIT"}},"version":1},"id":43} \ No newline at end of file +{"abi":[{"type":"function","name":"challengeWindow","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"getChallengesPerProof","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"getMaxProvingPeriod","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"initChallengeWindowStart","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"nextChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"challengeWindow()":"861a1412","getChallengesPerProof()":"47d3dfe7","getMaxProvingPeriod()":"f2f12333","initChallengeWindowStart()":"21918cea","nextChallengeWindowStart(uint256)":"8bf96d28"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"challengeWindow\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengesPerProof\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getMaxProvingPeriod\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"initChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"challengeWindow()\":{\"returns\":{\"_0\":\"Challenge window size in epochs\"}},\"getChallengesPerProof()\":{\"returns\":{\"_0\":\"Number of challenges required per proof\"}},\"getMaxProvingPeriod()\":{\"returns\":{\"_0\":\"Maximum proving period in epochs\"}},\"nextChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the data set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingWindow\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"challengeWindow()\":{\"notice\":\"Returns the number of epochs at the end of a proving period during which proofs can be submitted\"},\"getChallengesPerProof()\":{\"notice\":\"Returns the required number of challenges/merkle inclusion proofs per data set\"},\"getMaxProvingPeriod()\":{\"notice\":\"Returns the number of epochs allowed before challenges must be resampled\"},\"initChallengeWindowStart()\":{\"notice\":\"Value for initializing the challenge window start for any data set assuming proving period starts now\"},\"nextChallengeWindowStart(uint256)\":{\"notice\":\"Calculates the start of the next challenge window for a given data set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x4415a4694442f73ea1918b162168919946c877d2a4d5161a44230d0506b8866d\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://2604dd1dbbcb6e69f24ed41dea4fa9a86f0fe154e1ec8ebb146d130209fceab6\",\"dweb:/ipfs/QmRZzu99ZiYsFhdKbdDjenih15yKNYXuap42aRDq9XH1J2\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"pure","type":"function","name":"challengeWindow","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getChallengesPerProof","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getMaxProvingPeriod","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"initChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"challengeWindow()":{"returns":{"_0":"Challenge window size in epochs"}},"getChallengesPerProof()":{"returns":{"_0":"Number of challenges required per proof"}},"getMaxProvingPeriod()":{"returns":{"_0":"Maximum proving period in epochs"}},"nextChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the data set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"challengeWindow()":{"notice":"Returns the number of epochs at the end of a proving period during which proofs can be submitted"},"getChallengesPerProof()":{"notice":"Returns the required number of challenges/merkle inclusion proofs per data set"},"getMaxProvingPeriod()":{"notice":"Returns the number of epochs allowed before challenges must be resampled"},"initChallengeWindowStart()":{"notice":"Value for initializing the challenge window start for any data set assuming proving period starts now"},"nextChallengeWindowStart(uint256)":{"notice":"Calculates the start of the next challenge window for a given data set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x4415a4694442f73ea1918b162168919946c877d2a4d5161a44230d0506b8866d","urls":["bzz-raw://2604dd1dbbcb6e69f24ed41dea4fa9a86f0fe154e1ec8ebb146d130209fceab6","dweb:/ipfs/QmRZzu99ZiYsFhdKbdDjenih15yKNYXuap42aRDq9XH1J2"],"license":"Apache-2.0 OR MIT"}},"version":1},"id":48} \ No newline at end of file diff --git a/pdp/contract/PDPVerifier.abi b/pdp/contract/PDPVerifier.abi index 11c6256d8..e27d25856 100644 --- a/pdp/contract/PDPVerifier.abi +++ b/pdp/contract/PDPVerifier.abi @@ -71,7 +71,7 @@ }, { "type": "function", - "name": "MAX_PIECE_SIZE", + "name": "MAX_PIECE_SIZE_LOG2", "inputs": [], "outputs": [ { @@ -160,6 +160,19 @@ ], "stateMutability": "view" }, + { + "type": "function", + "name": "VERSION", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "string", + "internalType": "string" + } + ], + "stateMutability": "view" + }, { "type": "function", "name": "addPieces", @@ -172,24 +185,12 @@ { "name": "pieceData", "type": "tuple[]", - "internalType": "struct PDPVerifier.PieceData[]", + "internalType": "struct Cids.Cid[]", "components": [ { - "name": "piece", - "type": "tuple", - "internalType": "struct Cids.Cid", - "components": [ - { - "name": "data", - "type": "bytes", - "internalType": "bytes" - } - ] - }, - { - "name": "rawSize", - "type": "uint256", - "internalType": "uint256" + "name": "data", + "type": "bytes", + "internalType": "bytes" } ] }, @@ -230,7 +231,7 @@ "internalType": "uint256" } ], - "stateMutability": "view" + "stateMutability": "nonpayable" }, { "type": "function", @@ -274,6 +275,25 @@ ], "stateMutability": "payable" }, + { + "type": "function", + "name": "dataSetLive", + "inputs": [ + { + "name": "setId", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bool", + "internalType": "bool" + } + ], + "stateMutability": "view" + }, { "type": "function", "name": "deleteDataSet", @@ -311,7 +331,7 @@ { "name": "", "type": "tuple[]", - "internalType": "struct PDPVerifier.PieceIdAndOffset[]", + "internalType": "struct IPDPTypes.PieceIdAndOffset[]", "components": [ { "name": "pieceId", @@ -330,20 +350,7 @@ }, { "type": "function", - "name": "getChallengeFinality", - "inputs": [], - "outputs": [ - { - "name": "", - "type": "uint256", - "internalType": "uint256" - } - ], - "stateMutability": "view" - }, - { - "type": "function", - "name": "getChallengeRange", + "name": "getActivePieceCount", "inputs": [ { "name": "setId", @@ -353,7 +360,7 @@ ], "outputs": [ { - "name": "", + "name": "activeCount", "type": "uint256", "internalType": "uint256" } @@ -362,57 +369,71 @@ }, { "type": "function", - "name": "getFILUSDPrice", - "inputs": [], - "outputs": [ + "name": "getActivePieces", + "inputs": [ { - "name": "", - "type": "uint64", - "internalType": "uint64" + "name": "setId", + "type": "uint256", + "internalType": "uint256" }, { - "name": "", - "type": "int32", - "internalType": "int32" - } - ], - "stateMutability": "view" - }, - { - "type": "function", - "name": "getNextChallengeEpoch", - "inputs": [ + "name": "offset", + "type": "uint256", + "internalType": "uint256" + }, { - "name": "setId", + "name": "limit", "type": "uint256", "internalType": "uint256" } ], "outputs": [ { - "name": "", - "type": "uint256", - "internalType": "uint256" + "name": "pieces", + "type": "tuple[]", + "internalType": "struct Cids.Cid[]", + "components": [ + { + "name": "data", + "type": "bytes", + "internalType": "bytes" + } + ] + }, + { + "name": "pieceIds", + "type": "uint256[]", + "internalType": "uint256[]" + }, + { + "name": "rawSizes", + "type": "uint256[]", + "internalType": "uint256[]" + }, + { + "name": "hasMore", + "type": "bool", + "internalType": "bool" } ], "stateMutability": "view" }, { "type": "function", - "name": "getNextDataSetId", + "name": "getChallengeFinality", "inputs": [], "outputs": [ { "name": "", - "type": "uint64", - "internalType": "uint64" + "type": "uint256", + "internalType": "uint256" } ], "stateMutability": "view" }, { "type": "function", - "name": "getNextPieceId", + "name": "getChallengeRange", "inputs": [ { "name": "setId", @@ -512,10 +533,60 @@ }, { "type": "function", - "name": "getRandomness", + "name": "getFILUSDPrice", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + }, + { + "name": "", + "type": "int32", + "internalType": "int32" + } + ], + "stateMutability": "nonpayable" + }, + { + "type": "function", + "name": "getNextChallengeEpoch", "inputs": [ { - "name": "epoch", + "name": "setId", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getNextDataSetId", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "uint64", + "internalType": "uint64" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getNextPieceId", + "inputs": [ + { + "name": "setId", "type": "uint256", "internalType": "uint256" } @@ -584,6 +655,25 @@ ], "stateMutability": "view" }, + { + "type": "function", + "name": "getRandomness", + "inputs": [ + { + "name": "epoch", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "uint256", + "internalType": "uint256" + } + ], + "stateMutability": "view" + }, { "type": "function", "name": "getScheduledRemovals", @@ -616,6 +706,13 @@ "outputs": [], "stateMutability": "nonpayable" }, + { + "type": "function", + "name": "migrate", + "inputs": [], + "outputs": [], + "stateMutability": "nonpayable" + }, { "type": "function", "name": "nextProvingPeriod", @@ -654,12 +751,41 @@ }, { "type": "function", - "name": "dataSetLive", + "name": "pieceChallengable", + "inputs": [ + { + "name": "setId", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "pieceId", + "type": "uint256", + "internalType": "uint256" + } + ], + "outputs": [ + { + "name": "", + "type": "bool", + "internalType": "bool" + } + ], + "stateMutability": "view" + }, + { + "type": "function", + "name": "pieceLive", "inputs": [ { "name": "setId", "type": "uint256", "internalType": "uint256" + }, + { + "name": "pieceId", + "type": "uint256", + "internalType": "uint256" } ], "outputs": [ @@ -701,7 +827,7 @@ { "name": "proofs", "type": "tuple[]", - "internalType": "struct PDPVerifier.Proof[]", + "internalType": "struct IPDPTypes.Proof[]", "components": [ { "name": "leaf", @@ -739,54 +865,6 @@ "outputs": [], "stateMutability": "nonpayable" }, - { - "type": "function", - "name": "pieceChallengable", - "inputs": [ - { - "name": "setId", - "type": "uint256", - "internalType": "uint256" - }, - { - "name": "pieceId", - "type": "uint256", - "internalType": "uint256" - } - ], - "outputs": [ - { - "name": "", - "type": "bool", - "internalType": "bool" - } - ], - "stateMutability": "view" - }, - { - "type": "function", - "name": "pieceLive", - "inputs": [ - { - "name": "setId", - "type": "uint256", - "internalType": "uint256" - }, - { - "name": "pieceId", - "type": "uint256", - "internalType": "uint256" - } - ], - "outputs": [ - { - "name": "", - "type": "bool", - "internalType": "bool" - } - ], - "stateMutability": "view" - }, { "type": "function", "name": "schedulePieceDeletions", @@ -842,89 +920,70 @@ "stateMutability": "payable" }, { - "type": "function", - "name": "getActivePieceCount", + "type": "event", + "name": "ContractUpgraded", "inputs": [ { - "name": "setId", - "type": "uint256", - "internalType": "uint256" - } - ], - "outputs": [ + "name": "version", + "type": "string", + "indexed": false, + "internalType": "string" + }, { - "name": "", - "type": "uint256", - "internalType": "uint256" + "name": "implementation", + "type": "address", + "indexed": false, + "internalType": "address" } ], - "stateMutability": "view" + "anonymous": false }, { - "type": "function", - "name": "getActivePieces", + "type": "event", + "name": "DataSetCreated", "inputs": [ { "name": "setId", "type": "uint256", + "indexed": true, "internalType": "uint256" }, { - "name": "offset", + "name": "storageProvider", + "type": "address", + "indexed": true, + "internalType": "address" + } + ], + "anonymous": false + }, + { + "type": "event", + "name": "DataSetDeleted", + "inputs": [ + { + "name": "setId", "type": "uint256", + "indexed": true, "internalType": "uint256" }, { - "name": "limit", + "name": "deletedLeafCount", "type": "uint256", + "indexed": false, "internalType": "uint256" } ], - "outputs": [ - { - "name": "pieces", - "type": "tuple[]", - "internalType": "struct Cids.Cid[]", - "components": [ - { - "name": "data", - "type": "bytes", - "internalType": "bytes" - } - ] - }, - { - "name": "pieceIds", - "type": "uint256[]", - "internalType": "uint256[]" - }, - { - "name": "rawSizes", - "type": "uint256[]", - "internalType": "uint256[]" - }, - { - "name": "hasMore", - "type": "bool", - "internalType": "bool" - } - ], - "stateMutability": "view" + "anonymous": false }, { "type": "event", - "name": "Debug", + "name": "DataSetEmpty", "inputs": [ { - "name": "message", - "type": "string", - "indexed": false, - "internalType": "string" - }, - { - "name": "value", + "name": "setId", "type": "uint256", - "indexed": false, + "indexed": true, "internalType": "uint256" } ], @@ -989,7 +1048,7 @@ }, { "type": "event", - "name": "PossessionProven", + "name": "PiecesAdded", "inputs": [ { "name": "setId", @@ -998,29 +1057,17 @@ "internalType": "uint256" }, { - "name": "challenges", - "type": "tuple[]", + "name": "pieceIds", + "type": "uint256[]", "indexed": false, - "internalType": "struct PDPVerifier.PieceIdAndOffset[]", - "components": [ - { - "name": "pieceId", - "type": "uint256", - "internalType": "uint256" - }, - { - "name": "offset", - "type": "uint256", - "internalType": "uint256" - } - ] + "internalType": "uint256[]" } ], "anonymous": false }, { "type": "event", - "name": "ProofFeePaid", + "name": "PiecesRemoved", "inputs": [ { "name": "setId", @@ -1029,29 +1076,17 @@ "internalType": "uint256" }, { - "name": "fee", - "type": "uint256", - "indexed": false, - "internalType": "uint256" - }, - { - "name": "price", - "type": "uint64", - "indexed": false, - "internalType": "uint64" - }, - { - "name": "expo", - "type": "int32", + "name": "pieceIds", + "type": "uint256[]", "indexed": false, - "internalType": "int32" + "internalType": "uint256[]" } ], "anonymous": false }, { "type": "event", - "name": "DataSetCreated", + "name": "PossessionProven", "inputs": [ { "name": "setId", @@ -1060,42 +1095,66 @@ "internalType": "uint256" }, { - "name": "storageProvider", - "type": "address", - "indexed": true, - "internalType": "address" + "name": "challenges", + "type": "tuple[]", + "indexed": false, + "internalType": "struct IPDPTypes.PieceIdAndOffset[]", + "components": [ + { + "name": "pieceId", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "offset", + "type": "uint256", + "internalType": "uint256" + } + ] } ], "anonymous": false }, { "type": "event", - "name": "DataSetDeleted", + "name": "PriceOracleFailure", "inputs": [ { - "name": "setId", - "type": "uint256", - "indexed": true, - "internalType": "uint256" - }, - { - "name": "deletedLeafCount", - "type": "uint256", + "name": "reason", + "type": "bytes", "indexed": false, - "internalType": "uint256" + "internalType": "bytes" } ], "anonymous": false }, { "type": "event", - "name": "DataSetEmpty", + "name": "ProofFeePaid", "inputs": [ { "name": "setId", "type": "uint256", "indexed": true, "internalType": "uint256" + }, + { + "name": "fee", + "type": "uint256", + "indexed": false, + "internalType": "uint256" + }, + { + "name": "price", + "type": "uint64", + "indexed": false, + "internalType": "uint64" + }, + { + "name": "expo", + "type": "int32", + "indexed": false, + "internalType": "int32" } ], "anonymous": false @@ -1125,44 +1184,6 @@ ], "anonymous": false }, - { - "type": "event", - "name": "PiecesAdded", - "inputs": [ - { - "name": "setId", - "type": "uint256", - "indexed": true, - "internalType": "uint256" - }, - { - "name": "pieceIds", - "type": "uint256[]", - "indexed": false, - "internalType": "uint256[]" - } - ], - "anonymous": false - }, - { - "type": "event", - "name": "PiecesRemoved", - "inputs": [ - { - "name": "setId", - "type": "uint256", - "indexed": true, - "internalType": "uint256" - }, - { - "name": "pieceIds", - "type": "uint256[]", - "indexed": false, - "internalType": "uint256[]" - } - ], - "anonymous": false - }, { "type": "event", "name": "Upgraded", @@ -1272,4 +1293,4 @@ } ] } -] \ No newline at end of file +] diff --git a/pdp/contract/PDPVerifier.json b/pdp/contract/PDPVerifier.json index 6910d48b7..ed32895a5 100644 --- a/pdp/contract/PDPVerifier.json +++ b/pdp/contract/PDPVerifier.json @@ -1 +1 @@ -{"abi":[{"type":"constructor","inputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"BURN_ACTOR","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"EXTRA_DATA_MAX_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"FIL_USD_PRICE_FEED_ID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"LEAF_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_ENQUEUED_REMOVALS","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"MAX_PIECE_SIZE","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_CHALLENGE_SCHEDULED","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"NO_PROVEN_EPOCH","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"PYTH","inputs":[],"outputs":[{"name":"","type":"address","internalType":"contract IPyth"}],"stateMutability":"view"},{"type":"function","name":"RANDOMNESS_PRECOMPILE","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"SECONDS_IN_DAY","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"UPGRADE_INTERFACE_VERSION","inputs":[],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"},{"type":"function","name":"VERSION","inputs":[],"outputs":[{"name":"","type":"string","internalType":"string"}],"stateMutability":"view"},{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct IPDPTypes.PieceData[]","components":[{"name":"piece","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"rawSize","type":"uint256","internalType":"uint256"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"calculateProofFee","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"estimatedGasFee","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getActivePieceCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"activeCount","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getActivePieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"},{"name":"limit","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"pieces","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"rawSizes","type":"uint256[]","internalType":"uint256[]"},{"name":"hasMore","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getFILUSDPrice","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"},{"name":"","type":"int32","internalType":"int32"}],"stateMutability":"nonpayable"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"tuple","internalType":"struct Cids.Cid","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getRandomness","inputs":[{"name":"epoch","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"initialize","inputs":[{"name":"_challengeFinality","type":"uint256","internalType":"uint256"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"migrate","inputs":[],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"owner","inputs":[],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"proxiableUUID","inputs":[],"outputs":[{"name":"","type":"bytes32","internalType":"bytes32"}],"stateMutability":"view"},{"type":"function","name":"renounceOwnership","inputs":[],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"transferOwnership","inputs":[{"name":"newOwner","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"upgradeToAndCall","inputs":[{"name":"newImplementation","type":"address","internalType":"address"},{"name":"data","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"payable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"implementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"Initialized","inputs":[{"name":"version","type":"uint64","indexed":false,"internalType":"uint64"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"OwnershipTransferred","inputs":[{"name":"previousOwner","type":"address","indexed":true,"internalType":"address"},{"name":"newOwner","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"reason","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"Upgraded","inputs":[{"name":"implementation","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"error","name":"AddressEmptyCode","inputs":[{"name":"target","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967InvalidImplementation","inputs":[{"name":"implementation","type":"address","internalType":"address"}]},{"type":"error","name":"ERC1967NonPayable","inputs":[]},{"type":"error","name":"FailedCall","inputs":[]},{"type":"error","name":"IndexedError","inputs":[{"name":"idx","type":"uint256","internalType":"uint256"},{"name":"msg","type":"string","internalType":"string"}]},{"type":"error","name":"InvalidInitialization","inputs":[]},{"type":"error","name":"NotInitializing","inputs":[]},{"type":"error","name":"OwnableInvalidOwner","inputs":[{"name":"owner","type":"address","internalType":"address"}]},{"type":"error","name":"OwnableUnauthorizedAccount","inputs":[{"name":"account","type":"address","internalType":"address"}]},{"type":"error","name":"UUPSUnauthorizedCallContext","inputs":[]},{"type":"error","name":"UUPSUnsupportedProxiableUUID","inputs":[{"name":"slot","type":"bytes32","internalType":"bytes32"}]}],"bytecode":{"object":"0x60a08060405234620000d157306080527ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a009081549060ff8260401c16620000c257506001600160401b036002600160401b0319828216016200007c575b604051615d359081620000d682396080518181816129710152612ae50152f35b6001600160401b031990911681179091556040519081527fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d290602090a15f80806200005c565b63f92ee8a960e01b8152600490fd5b5f80fdfe610120806040526004361015610013575f80fd5b5f905f3560e01c908163029b464614613ee55750806304595c1a14613e915780630a6a63f114613e455780630c29202414613ac65780630cd7b88014613a8d57806315b1757014613a4057806319c75950146139e75780631a271225146139cd5780631c5ae80f1461397957806321b7cd1c146138ff57806325bbbedf146138c95780632b3129bb1461385f578063349c9179146137a957806339f51544146136b2578063431860801461351c578063442cded3146134d6578063453f4f621461349957806345c0b92d14612da5578063462dd449146110655780634903704a14612d305780634f1ef28614612a3d5780634fa27920146129eb57806352d1902d1461292b5780635353bdfd1461288e57806361a52a361461285257806367e406d5146128055780636ba4608f146127b15780636fa4469214612717578063715018a6146126595780637a1e29901461240057806389208ba9146123ac5780638a405abc1461236c5780638da5cb5b146122fb5780638fd3ab80146121a35780639f8cb3bd14612168578063a531998c14612114578063ad3cb1cc14612098578063bbae41cb14611db0578063c0e1594914611d77578063ca759f2714611d30578063dc63526614611b31578063ddea76cc14611277578063df0f32481461106a578063f178b1be14611065578063f2fde38b1461101a578063f58f952b1461047b578063f83758fe14610440578063fe4b84df1461028e5763ffa1ad741461023a575f80fd5b3461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876102736142b4565b60405191829160208352602083019061401e565b0390f35b80fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b577ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c16159067ffffffffffffffff811680159081610438575b600114908161042e575b159081610425575b506103fb578160017fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000083161784556103c6575b5061034c61583c565b61035461583c565b61035d33614fc6565b61036561583c565b6004358355610372575080f35b7fffffffffffffffffffffffffffffffffffffffffffffff00ffffffffffffffff81541690557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160018152a180f35b7fffffffffffffffffffffffffffffffffffffffffffffff00000000000000000016680100000000000000011782555f610343565b60046040517ff92ee8a9000000000000000000000000000000000000000000000000000000008152fd5b9050155f610310565b303b159150610308565b8391506102fe565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760209054604051908152f35b506104853661407a565b9291905a93828452600b60205273ffffffffffffffffffffffffffffffffffffffff92836040862054163303610f96578115610f385780855260076020526040852054804310610eda5715610e7c576104dd82614625565b9381865260076020526104f36040872054614b55565b8287526009602052604087205490600560205261051360408920546150db565b9261010084810311610e4f579193888888888d9484985b8267ffffffffffffffff8b161015610ac6576040518860208201528260408201527fffffffffffffffff0000000000000000000000000000000000000000000000008b60c01b1660608201526048815280608081011067ffffffffffffffff608083011117610a995760808101604052602081519101208b15610a6c576105b9908c83610100039106846152ac565b6105cd67ffffffffffffffff8c16876146a0565b526105e267ffffffffffffffff8b16866146a0565b506106026105fa67ffffffffffffffff8c16876146a0565b515183614503565b602081515110610a0e5760405190610619826141cf565b6020825260203681840137875b602081106109455750508660c0526020815191015160c0526020811061090f575b5081865260036020526040862061066867ffffffffffffffff8c16876146a0565b5151875260205260408620547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff81019081116108e2576106a7906150db565b610100039a6101008c116108e25760018c018c116108e2578a9b8b602061070567ffffffffffffffff6106fc816106ee6106e58d8d848a1691614f86565b86810190615087565b906101005295168a8c614f86565b359f168a6146a0565b5101516080526107148161460d565b60e05260405160a05261072b60e05160a0516141eb565b60a051508060a05152602060a05101368260051b6101005101116108de5761010051905b8260051b610100510182106108ce5750505060a051510361084a57608051969a96988b975b60a051518d10156107bd5760019061078e8e60a0516146a0565b51908c83166107ae57906107a191615cc4565b9a5b811c9c019b99610774565b6107b791615cc4565b9a6107a3565b9195995093979b91959992969a5060c051036107ec576107dc90614ec6565b989490979399959196929961052a565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f70726f6f6620646964206e6f74207665726966790000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f70726f6f66206c656e67746820646f6573206e6f74206d61746368207472656560448201527f20686569676874000000000000000000000000000000000000000000000000006064820152fd5b813581526020918201910161074f565b8980fd5b6024877f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9060200360031b1b60c0511660c0528b610647565b81518051807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101116109e1576109cc83926109c66001957fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe07fff0000000000000000000000000000000000000000000000000000000000000095016143c5565b9061582b565b51168a1a6109da828661582b565b5301610626565b60248b7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601560248201527f436964206461746120697320746f6f2073686f727400000000000000000000006044820152fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b5093959091610ad788965a90614600565b879188905b858210610e0057505060208201809211610dd35761051491828102928184041490151715610dd357610b1891610b11916143c5565b4890614792565b84875260096020526040872054907f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82168203610dd357610b83610b799493926040928a610b64614d26565b988196838d8c9552600d602052205443614600565b9360051b926154c3565b803410610d75575f8080808473ff000000000000000000000000000000000000635af1610bae614b26565b5015610d1757867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b6060610c03948c9867ffffffffffffffff6040519287845216602083015260030b6040820152a234614600565b9585845260086020526040842054169081610c84575b5050507f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e509183610c5b9252600d602052436040872055604051918291826140c8565b0390a28181610c675780f35b808080610c7f94335af1610c79614b26565b50614edf565b818180f35b6006602052604084205491803b15610d135784928360849260405196879586947f356de02b0000000000000000000000000000000000000000000000000000000086528c60048701526024860152604485015260648401525af18015610d0857610cf0575b8080610c19565b610cf99061419f565b610d04578385610ce9565b8380fd5b6040513d84823e3d90fd5b8480fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f4275726e206661696c65640000000000000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f496e636f72726563742066656520616d6f756e740000000000000000000000006044820152fd5b6024887f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b9092610e1a610e10858885614f86565b6020810190615087565b80915060051b90808204602014901517156109e15760400190816040116109e157600191610e47916143c5565b930190610adc565b6024897f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601660248201527f6e6f206368616c6c656e6765207363686564756c6564000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600f60248201527f7072656d61747572652070726f6f6600000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f656d7074792070726f6f660000000000000000000000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20707260448201527f6f766520706f7373657373696f6e0000000000000000000000000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611062611055614144565b61105d6157a5565b614fc6565b80f35b614167565b503461028b5761107936614266565b919061108c61108783614f44565b6142ed565b818452600c60205273ffffffffffffffffffffffffffffffffffffffff918260408620541633036111cd578493818552600b60205260408520928354858116947fffffffffffffffffffffffff00000000000000000000000000000000000000009182339116179055600c60205260408720908154169055604051943385857f686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c98a80a460086020526040872054169182611144578680f35b823b156111c9578561119d819593899793889484967f4059b6d700000000000000000000000000000000000000000000000000000000865260048601526024850152336044850152608060648501526084840191614424565b03925af18015610d08576111b5575b80808080808680f35b6111be9061419f565b61028b57805f6111ac565b8680fd5b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f6e6c79207468652070726f706f7365642073746f726167652070726f76696460448201527f65722063616e20636c61696d2073746f726167652070726f766964657220726f60648201527f6c650000000000000000000000000000000000000000000000000000000000006084820152fd5b503461028b5761128636613f7c565b61129861080082969593961115614352565b6112a461108785614f44565b8215611ad357838652600b60205273ffffffffffffffffffffffffffffffffffffffff6040872054163303611a4f578386526005602052604086205494846112eb8561472b565b885b868110611569575061132e7fd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed85991604051918291602083526020830190614111565b0390a2848752600860205273ffffffffffffffffffffffffffffffffffffffff6040882054169384611366575b602087604051908152f35b843b156115655794929091879492866040519788967f545f6ec5000000000000000000000000000000000000000000000000000000008852608488019060048901528a6024890152608060448901525260a486019060a48160051b88010194809289915b83831061144b57505050505061140f8387938795937ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc87809603016064860152614424565b03925af1801561144057611428575b808080808061135b565b611432839161419f565b61143c578161141e565b5080fd5b6040513d85823e3d90fd5b929597995092977fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5c9087929597030183528735907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc18336030182121561156157828201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18085850136030182121561155d576040835284840182013603018484018201351215611559578284010180350167ffffffffffffffff8135116115595780353603602082011361155957600192826020808761153e8297968360408199015260608601908481359101614424565b94010135910152990193019301899795938c999795926113ca565b8d80fd5b8e80fd5b8c80fd5b8780fd5b909150611577818787614f86565b8035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe181360301821215611a4b5760206115b3848a8a614f86565b013591601f83166119e657821561198157660400000000000083116118f657908b9392918a8552600560205260408520918254926115f0846147a5565b90556116036115fe846143b7565b615935565b8c8560051c9188915b8183106118b057915050875260046020526040872084885260205260408720558b86526002602052604086208387526020526040862091808201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18284013603018112156115655767ffffffffffffffff8183850101351161156557828201810180353603602090910113611565576116a684546144b2565b601f8111611871575b508790601f818486010135116001146117925760019891818486010135611760575b5082817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9394860101358a1b9401013560031b1c19161790555b898c52600360205260408c20908c526020528060051c60408c2055888b52600660205261174060408c209160051c82546143c5565b905561174c818a6143c5565b61175682856146a0565b52019086916112ed565b8484018201016020013591507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6116d1565b848952602089209092915b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0848487010135168a1061185357600199508483018401357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe081161061180f575b50508792010135811b01905561170b565b60207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60f886868901013560031b161c199185858801010101351690555f806117fe565b6020838601850182018101358355998a01996001909201910161179d565b8489526020808a206118a092868601850135601f810160051c830193116118a6575b601f0160051c0190615815565b5f6116af565b9091508190611893565b9260408395969798999a6118e693956118cd600180971b8c614600565b90825260046020528282209082526020522054906143c5565b92018f979695949392918e61160c565b60a484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152602160448201527f50696563652073697a65206d757374206265206c657373207468616e20325e3560648201527f30000000000000000000000000000000000000000000000000000000000000006084820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601b60448201527f53697a65206d7573742062652067726561746572207468616e203000000000006064820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601d60448201527f53697a65206d7573742062652061206d756c7469706c65206f662033320000006064820152fd5b8a80fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20616460448201527f64207069656365730000000000000000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601b60248201527f4d75737420616464206174206c65617374206f6e6520706965636500000000006044820152fd5b503461028b57611b4036613fea565b818392935260209260058452611b5960408420546150db565b610100908103908111611d0357818452600985526040842054907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91828101908111611cd65790611baa91846152ac565b93858501519083815260038752604081208651825287526040812054928301928311611ca9575003611bff5781611be091614462565b9182611bf3575b50506040519015158152f35b51101590505f80611be7565b60a484604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152604160248201527f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e207760448201527f697468207468652076657279206c617374206c656166206f662061207069656360648201527f65000000000000000000000000000000000000000000000000000000000000006084820152fd5b807f4e487b7100000000000000000000000000000000000000000000000000000000602492526011600452fd5b6024867f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b6024847f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020611d6d600435614f44565b6040519015158152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051818152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611de3614144565b9067ffffffffffffffff9060243582811161143c57611e06903690600401613f4e565b9390611e16610800861115614352565b67016345785d8a00009182341061203a575f8080808673ff000000000000000000000000000000000000635af1611e4b614b26565b5015610d17576001547fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000086821696611e8288614ec6565b169116176001558484526020956006875284604081205560078752846040812055600b875260408520917fffffffffffffffffffffffff00000000000000000000000000000000000000009233848254161790556008885273ffffffffffffffffffffffffffffffffffffffff6040872091168093825416179055600d875284604081205581611fb6575b50505033837f11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b962818480a33411611f46575b50604051908152f35b7ffffffffffffffffffffffffffffffffffffffffffffffffffe9cba87a2760000340190348211611f8957808080611f8394335af1610c79614b26565b5f611f3d565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b813b15610d135791849161200e93836040518096819582947f101c1eab0000000000000000000000000000000000000000000000000000000084528c6004850152336024850152606060448501526064840191614424565b03925af1801561144057908391612026575b80611f0d565b61202f9061419f565b61143c57815f612020565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f737962696c20666565206e6f74206d65740000000000000000000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876040516120d6816141cf565b600581527f352e302e30000000000000000000000000000000000000000000000000000000602082015260405191829160208352602083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57604060209160043561215861108782614f44565b8152600683522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040516107d08152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576121da6157a5565b7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c1680156122e6575b6103fb577fffffffffffffffffffffffffffffffffffffffffffffff0000000000000000006002917f2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c36122ab61225f6142b4565b73ffffffffffffffffffffffffffffffffffffffff7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541660405192839260408452604084019061401e565b9060208301520390a1161790557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160028152a180f35b50600267ffffffffffffffff8216101561220b565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602073ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993005416604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405166040000000000008152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356123f061108782614f44565b8152600983522054604051908152f35b503461028b5761240f36614266565b9061241e610800831115614352565b67ffffffffffffffff600154168310156125fb57828452602090600b825273ffffffffffffffffffffffffffffffffffffffff92836040872054163303612577579085918583526006845260408320948386549655600b8552604084207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905560078552836040812055600d85528360408120556008855260408420541691826124f6575b505050507f14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb791604051908152a280f35b823b15610d045761254d928492836040518096819582947f2abd465c0000000000000000000000000000000000000000000000000000000084528d60048501528c6024850152606060448501526064840191614424565b03925af18015610d0857612563575b80806124c6565b61256c9061419f565b610d0457835f61255c565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20646560448201527f6c657465206461746120736574730000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601960248201527f6461746120736574206964206f7574206f6620626f756e6473000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576126906157a5565b5f73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993008054907fffffffffffffffffffffffff000000000000000000000000000000000000000082169055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e08280a380f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043561275661108782614f44565b8152600a6020526040812080549061276d8261472b565b925b82811061278c576040516020808252819061028790820187614111565b806127996001928461440f565b90549060031b1c6127aa82876146a0565b520161276f565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356127f561108782614f44565b8152600783522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173a2aa501b19aff244d90cc15a4cf739d2725b57298152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051620151808152f35b503461028b5760209060207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576004359080926128d361108784614f44565b8282526005602052604082205492825b8481106128f557602086604051908152f35b818452600383526040842081855283526040842054612917575b6001016128e3565b946129236001916147a5565b95905061290f565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff7f00000000000000000000000000000000000000000000000000000000000000001630036129c15760206040517f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8152f35b60046040517fe07c8dba000000000000000000000000000000000000000000000000000000008152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576040612a24614d26565b67ffffffffffffffff83519216825260030b6020820152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57612a70614144565b602491823567ffffffffffffffff811161143c573660238201121561143c57806004013592612a9e8461422c565b612aab60405191826141eb565b8481526020948582019336888383010111612cf7578186928989930187378301015273ffffffffffffffffffffffffffffffffffffffff807f000000000000000000000000000000000000000000000000000000000000000016803014908115612d02575b506129c157612b1d6157a5565b821694604051907f52d1902d00000000000000000000000000000000000000000000000000000000825280826004818a5afa9182918793612cce575b5050612b8f578686604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b8590877f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc91828103612ca05750843b15612c71575080547fffffffffffffffffffffffff000000000000000000000000000000000000000016821790556040518592917fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b8480a2815115612c3b5750612c379382915190845af4612c31614b26565b91615895565b5080f35b935050505034612c49575080f35b807fb398979f0000000000000000000000000000000000000000000000000000000060049252fd5b82604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b604051907faa1d49a40000000000000000000000000000000000000000000000000000000082526004820152fd5b9080929350813d8311612cfb575b612ce681836141eb565b81010312612cf75751905f80612b59565b8580fd5b503d612cdc565b9050817f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541614155f612b10565b503461028b57612d3f36613fea565b81835260096020526040832054917f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83168303611d035790602093612d9d9392610b796040612d8c614d26565b9490938152600d8952205443614600565b604051908152f35b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760443567ffffffffffffffff811161143c57612df5903690600401613f4e565b90612e04610800831115614352565b6004358352600b60205273ffffffffffffffffffffffffffffffffffffffff60408420541633036134155760043583526006602052604083205415613391576004358352600d60205260408320805415613388575b50600a602052604083208054806130b5575b505060043583526006602052604083205460096020526040842055612e918354436143c5565b6024351061300b5782916004358352600760205260243560408420556006602052604083205415612fc9575b600860205273ffffffffffffffffffffffffffffffffffffffff6040842054169081612f2d575b8360043581526006602052604081205460405190602435825260208201527fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e604060043592a280f35b600760205260408420549160066020526040852054813b15612cf757858094612f9f604051978896879586947faa27ebcc000000000000000000000000000000000000000000000000000000008652600435600487015260248601526044850152608060648501526084840191614424565b03925af18015610d0857612fb5575b8080612ee4565b612fbe9061419f565b61028b57805f612fae565b6004357f02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c28428480a2600d6020528260408120556007602052826040812055612ebd565b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f6368616c6c656e67652065706f6368206d757374206265206174206c6561737460448201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560648201527f20667574757265000000000000000000000000000000000000000000000000006084820152fd5b6130be8161472b565b91855b8281106132e75750505090916130db611087600435614f44565b83805b8351821015613282576130f182856146a0565b5194600435875260036020526040872086885260205260408720549386916004358952600560205261312660408a20546150db565b9661010088810311613255579261313f6115fe8a6143b7565b88610100038111158061323e575b1561319b57906001613186926004358d5260046020528c8360408220915260205260408d2061317d8b8254614600565b90551b906143c5565b926131936115fe856143b7565b93909361313f565b505095509295936001926131f592956004358a52600360205260408a20818b526020525f60408b20556004358a52600260205260408a20908a5260205288604081206131e781546144b2565b80613200575b5050506143c5565b9401909192936130de565b601f808211881461321a5750505f9150555b885f806131ed565b916132375f929382865260208620940160051c8401898501615815565b5555613212565b506004358b52600560205260408b2054821061314d565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b905092919092600435855260066020526132a160408620918254614600565b90557f6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b160405160208152806132dd600435946020830190614111565b0390a25f80612e6b565b81547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808201918083116132555761331f838661440f565b919054600392831b1c613332868a6146a0565b521561335b57908291613348600195948761440f565b81939154921b1b191690558355016130c1565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526031600452fd5b4390555f612e59565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602c60248201527f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c65617660448201527f65732061726520616464656400000000000000000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603960248201527f6f6e6c79207468652073746f726167652070726f76696465722063616e206d6f60448201527f766520746f206e6578742070726f76696e6720706572696f64000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020612d9d600435614b55565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602067ffffffffffffffff60015416604051908152f35b503461028b5760407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043560243573ffffffffffffffffffffffffffffffffffffffff8082168092036136ae5761357d61108784614f44565b828452600b6020526040842054163381036136045781036135cd57508152600c602052604081207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905580f35b908252600c60205260408220907fffffffffffffffffffffffff000000000000000000000000000000000000000082541617905580f35b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604460248201527f4f6e6c79207468652063757272656e742073746f726167652070726f7669646560448201527f722063616e2070726f706f73652061206e65772073746f726167652070726f7660648201527f69646572000000000000000000000000000000000000000000000000000000006084820152fd5b5f80fd5b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576136f36044356024356004356147d2565b9160409491945193608085019260808652815180945260a086019360a08160051b88010194602080940192905b82821061375957888703858a015288808961374d8d61373f8c8c614111565b908482036040860152614111565b90151560608301520390f35b90919295848061379b837fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff608d60019603018652828b515191818152019061401e565b980192019201909291613720565b503461028b576137b83661407a565b909180845260056020526137cf60408520546150db565b91610100928303928311613832576137e681614625565b945b8181106137fd576040518061028788826140c8565b806138168561380f600194868a6143d2565b35866152ac565b61382082896146a0565b5261382b81886146a0565b50016137e8565b6024857f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff60406020926004356138b861108782614f44565b815260088452205416604051908152f35b503461028b576102876138e46138de36613fea565b90614503565b6040519182916020835251602080840152604083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760409060043561394161108782614f44565b8152600b60205273ffffffffffffffffffffffffffffffffffffffff8281818420541692600c60205220541682519182526020820152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356139bd61108782614f44565b8152600583522054604051908152f35b503461028b576020611d6d6139e136613fea565b90614462565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040517f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e8152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173fe000000000000000000000000000000000000068152f35b503461028b576040602091613aa136613fea565b90613aae61108782614f44565b82526003845282822090825283522054604051908152f35b50346136ae57613ad536613f7c565b90613ae7610800839594951115614352565b613af361108786614f44565b845f526020600b815273ffffffffffffffffffffffffffffffffffffffff908160405f2054163303613dc157865f52600a8082526107d0613b3860405f2054896143c5565b11613d3d575f5b878110613c26575050600890875f525260405f2054169182613b5f578680f35b823b156136ae57604051957fe7954aa70000000000000000000000000000000000000000000000000000000087526004870152606060248701528460648701527f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff85116136ae5785613bf38195935f9793608484968a9660051b809183880137850160808682030160448701520191614424565b03925af18015613c1b57613c0a5780808080808680f35b613c14915061419f565b5f806111ac565b6040513d5f823e3d90fd5b613c318189896143d2565b35895f526005845260405f20541115613cb957885f5281835260405f2090613c5a818a8a6143d2565b3582549268010000000000000000841015610a995783613c80916001809601815561440f565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff829392549160031b92831b921b191617905501613b3f565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602c60248201527f43616e206f6e6c79207363686564756c652072656d6f76616c206f662065786960448201527f7374696e672070696563657300000000000000000000000000000000000000006064820152fd5b608482604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603a60248201527f546f6f206d616e792072656d6f76616c73207761697420666f72206e6578742060448201527f70726f76696e6720706572696f6420746f207363686564756c650000000000006064820152fd5b608490604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20736360448201527f686564756c652072656d6f76616c206f662070696563657300000000000000006064820152fd5b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57602060405173ff000000000000000000000000000000000000638152f35b346136ae5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57600435613ecf61108782614f44565b5f52600d602052602060405f2054604051908152f35b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae578061080060209252f35b9181601f840112156136ae5782359167ffffffffffffffff83116136ae576020808501948460051b0101116136ae57565b9181601f840112156136ae5782359167ffffffffffffffff83116136ae57602083818601950101116136ae57565b9060607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae576004359167ffffffffffffffff916024358381116136ae5782613fcc91600401613f1d565b939093926044359182116136ae57613fe691600401613f4e565b9091565b7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc60409101126136ae576004359060243590565b91908251928382525f5b8481106140665750507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f845f6020809697860101520116010190565b602081830181015184830182015201614028565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f1d565b60208082019080835283518092528060408094019401925f905b8382106140f157505050505090565b8451805187528301518684015294850194938201936001909101906140e2565b9081518082526020808093019301915f5b828110614130575050505090565b835185529381019392810192600101614122565b6004359073ffffffffffffffffffffffffffffffffffffffff821682036136ae57565b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae5760206040515f8152f35b67ffffffffffffffff8111610a9957604052565b6020810190811067ffffffffffffffff821117610a9957604052565b6040810190811067ffffffffffffffff821117610a9957604052565b90601f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0910116810190811067ffffffffffffffff821117610a9957604052565b67ffffffffffffffff8111610a9957601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01660200190565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f4e565b604051906142c1826141cf565b600582527f322e302e300000000000000000000000000000000000000000000000000000006020830152565b156142f457565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f4461746120736574206e6f74206c6976650000000000000000000000000000006044820152fd5b1561435957565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f4578747261206461746120746f6f206c617267650000000000000000000000006044820152fd5b9060018201809211611f8957565b91908201809211611f8957565b91908110156143e25760051b0190565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b80548210156143e2575f5260205f2001905f90565b601f82602094937fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe093818652868601375f8582860101520116010190565b9061446c82614f44565b918261449b575b8261447d57505090565b9091505f52600360205260405f20905f5260205260405f2054151590565b8092505f52600560205260405f2054811091614473565b90600182811c921680156144f9575b60208310146144cc57565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b91607f16916144c1565b6060604051614511816141b3565b5261451e61108782614f44565b5f526020906002825260405f20905f52815260405f2060405191614541836141b3565b60405180925f908054614553816144b2565b808552916001918083169081156145bf5750600114614581575b50505061457c925003826141eb565b815290565b5f90815285812095935091905b8183106145a757505061457c93508201015f808061456d565b8554878401850152948501948694509183019161458e565b91505061457c9593507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f808061456d565b91908203918211611f8957565b67ffffffffffffffff8111610a995760051b60200190565b9061462f8261460d565b60409061463f60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe061466d829561460d565b01915f5b83811061467e5750505050565b602090825161468c816141cf565b5f8152825f81830152828601015201614671565b80518210156143e25760209160051b010190565b906146be8261460d565b6040906146ce60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe06146fc829561460d565b01915f5b83811061470d5750505050565b602090825161471b816141b3565b6060815282828601015201614700565b906147358261460d565b61474260405191826141eb565b8281527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0614770829461460d565b0190602036910137565b906701518000000000009180830292830403611f8957565b81810292918115918404141715611f8957565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8114611f895760010190565b915f6147e061108785614f44565b8115614ac857835f52600560205260405f2054926147fd836146b4565b926148078161472b565b946148118261472b565b965f905f945f5b848110614905575b505050505081155f1461486d5750505050505060405161483f816141b3565b5f81526040519161484f836141b3565b5f83526040519161485f836141b3565b5f83525f3681379291905f90565b81969293949596105f146148ff57614884816146b4565b9561488e8261472b565b956148988361472b565b955f5b8481106148a9575050505050565b806148b6600192846146a0565b516148c1828d6146a0565b526148cc818c6146a0565b506148d781856146a0565b516148e2828c6146a0565b526148ed81866146a0565b516148f8828b6146a0565b520161489b565b50919391565b825f52600360205260405f20815f5260205260405f2054614929575b600101614818565b92958187101580614abf575b15614a8e57825f52600260205260405f20845f5260205260405f206040519061495d826141b3565b60405190815f82549261496f846144b2565b8084529360018116908115614a4e5750600114614a0c575b50614994925003826141eb565b81526149a0828b6146a0565b526149ab818a6146a0565b50836149b7828c6146a0565b52825f52600360205260405f20845f5260205260405f2054908160051b9180830460201490151715611f89576149fe816001936149f88f94614a04956146a0565b526147a5565b976147a5565b939050614921565b9150505f528160205f20915f925b818410614a325750506020614994928201015f614987565b6020919250806001915483868801015201920191908391614a1a565b602093506149949592507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f614987565b95614a9986836143c5565b811015614aab57614a046001916147a5565b505050505091506001915f80808080614820565b50858110614935565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601c60248201527f4c696d6974206d7573742062652067726561746572207468616e2030000000006044820152fd5b3d15614b50573d90614b378261422c565b91614b4560405193846141eb565b82523d5f602084013e565b606090565b5f80916040516020810191825260208152614b6f816141cf565b519073fe000000000000000000000000000000000000065afa614b90614b26565b9015614baa576020818051810103126136ae576020015190565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c6560448201527f64000000000000000000000000000000000000000000000000000000000000006064820152fd5b908160809103126136ae576040519067ffffffffffffffff6080830181811184821017610a995760405281518060070b81036136ae578352602082015190811681036136ae5760208301526040810151908160030b82036136ae5760609160408401520151606082015290565b15614ca257565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f6661696c656420746f2076616c69646174653a207072696365206d757374206260448201527f652067726561746572207468616e2030000000000000000000000000000000006064820152fd5b6040908151917fa4ae35e00000000000000000000000000000000000000000000000000000000083527f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e9283600482015262015180602482015260809373a2aa501b19aff244d90cc15a4cf739d2725b5729918581604481865afa5f9181614ea7575b50614e92575084907f32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120614df0614ddd614b26565b865191829160208352602083019061401e565b0390a160248451809481937f96834ad300000000000000000000000000000000000000000000000000000000835260048301525afa938415614e88575f94614e59575b5050614e445f845160070b13614c9b565b825167ffffffffffffffff1692015160030b90565b614e79929450803d10614e81575b614e7181836141eb565b810190614c2e565b915f80614e33565b503d614e67565b82513d5f823e3d90fd5b809550614e4492505f91505160070b13614c9b565b614ebf919250873d8911614e8157614e7181836141eb565b905f614da9565b67ffffffffffffffff809116908114611f895760010190565b15614ee657565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601060248201527f5472616e73666572206661696c65642e000000000000000000000000000000006044820152fd5b67ffffffffffffffff6001541681109081614f5d575090565b90505f52600b60205273ffffffffffffffffffffffffffffffffffffffff60405f205416151590565b91908110156143e25760051b810135907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc1813603018212156136ae570190565b73ffffffffffffffffffffffffffffffffffffffff809116908115615057577f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300805490837fffffffffffffffffffffffff00000000000000000000000000000000000000008316179055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e05f80a3565b60246040517f1e4fbdf70000000000000000000000000000000000000000000000000000000081525f6004820152fd5b9035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe1813603018212156136ae570180359067ffffffffffffffff82116136ae57602001918160051b360383136136ae57565b610100908060801c806152a0575b508060401c8061526d575b508060201c8061523a575b508060101c80615207575b508060081c806151d4575b508060041c806151a1575b508060021c8061516e575b508060011c6151405761513d91614600565b90565b507ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe810191508111611f8957905f61512b565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc810191508111611f8957905f615120565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8810191508111611f8957905f615115565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0810191508111611f8957905f61510a565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0810191508111611f8957905f6150ff565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc0810191508111611f8957905f6150f4565b9150506080905f6150e9565b916040918251916152bc836141cf565b5f83525f602080940152845f5260068352835f2054821015615452576001947fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86831b818101908111611f895796905f9391825b61536e575050505f5260048352835f20855f528352615332845f2054826143c5565b82811115615358575061534491614600565b915192615350846141cf565b835282015290565b905060018501809511611f895761534491614600565b909197835f5260058752875f205481101561543b57835f52600494858852885f20825f528852866153a28a5f2054836143c5565b1161541b576153c390855f52868952895f20835f528952895f2054906143c5565b94838a01908a82116153ef575090826153dd921b906143c5565b975b8015611f89578201919082615310565b6011907f4e487b71000000000000000000000000000000000000000000000000000000005f525260245ffd5b94838a01908a82116153ef57509082615435921b90614600565b976153df565b82890190898211611f895782615435921b90614600565b6064838551907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152601860248201527f4c65616620696e646578206f7574206f6620626f756e647300000000000000006044820152fd5b63ffffffff16604d8111611f8957600a0a90565b93929190841580159061579d575b156157195767ffffffffffffffff1690811561569557821561561157600381900b5f81126155a3575061551561550961551b9361477a565b9163ffffffff166154af565b90614792565b8015610a6c5761553d9261553891671bc16d674ec8000004614792565b614792565b9060058202821590838104600514821715611f895760649004908360021b848104600414821715611f89578284106155785750505050505f90565b60649004831061558d575061513d9250614600565b9150508180046001141715611f89576064900490565b90507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008114611f89576155dd905f0363ffffffff166154af565b90671bc16d674ec800009180830292830403611f89576155fc9061477a565b928315610a6c5761553d936155389204614792565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603360248201527f6661696c656420746f2076616c69646174653a207261772073697a65206d757360448201527f742062652067726561746572207468616e2030000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603860248201527f6661696c656420746f2076616c69646174653a204174746f46494c207072696360448201527f65206d7573742062652067726561746572207468616e203000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603c60248201527f6661696c656420746f2076616c69646174653a20657374696d6174656420676160448201527f7320666565206d7573742062652067726561746572207468616e2030000000006064820152fd5b5048156154d1565b73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300541633036157e557565b60246040517f118cdaa7000000000000000000000000000000000000000000000000000000008152336004820152fd5b818110615820575050565b5f8155600101615815565b9081518110156143e2570160200190565b60ff7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a005460401c161561586b57565b60046040517fd7e6bcf8000000000000000000000000000000000000000000000000000000008152fd5b906158d457508051156158aa57805190602001fd5b60046040517fd6bda275000000000000000000000000000000000000000000000000000000008152fd5b8151158061592c575b6158e5575090565b60249073ffffffffffffffffffffffffffffffffffffffff604051917f9996b315000000000000000000000000000000000000000000000000000000008352166004820152fd5b50803b156158dd565b7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8111615c4057610100907f80000000000000000000000000000000000000000000000000000000000000008114611f8957805f031680615c37575b6fffffffffffffffffffffffffffffffff8116615c06575b77ffffffffffffffff0000000000000000ffffffffffffffff8116615bd5575b7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff8116615ba4575b7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff8116615b73575b7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff8116615b42575b7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f8116615b11575b7f33333333333333333333333333333333333333333333333333333333333333338116615ae0575b7f555555555555555555555555555555555555555555555555555555555555555516615ab35790565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8101908111611f895790565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790615a8a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8101908111611f895790615a62565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff88101908111611f895790615a3a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff08101908111611f895790615a13565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101908111611f8957906159ed565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc08101908111611f8957906159c9565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808101908111611f8957906159a9565b60ff9150615991565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f496e7075742065786365656473206d6178696d756d20696e743235362076616c60448201527f75650000000000000000000000000000000000000000000000000000000000006064820152fd5b5f5260205260205f60408160025afa156136ae577fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f5f51169056fea2646970667358221220a2d6c54c44a68038f7d59aa211cb3b39a5f231a0e7b9f3f652141521dc58b6fd64736f6c63430008170033","sourceMap":"2048:37005:43:-:0;;;;;;;1171:4:25;1163:13;;8837:64:24;2048:37005:43;;;;;;;;;7896:76:24;;-1:-1:-1;;;;;;;;;;;;2048:37005:43;;;7985:34:24;7981:146;;-1:-1:-1;2048:37005:43;;;;;;;;1163:13:25;2048:37005:43;;;;;;;;;;;7981:146:24;-1:-1:-1;;;;;;2048:37005:43;;;;;;;;;;;;;8087:29:24;;2048:37005:43;;8087:29:24;7981:146;;;;;7896:76;-1:-1:-1;;;7938:23:24;;;;;2048:37005:43;;;","linkReferences":{}},"deployedBytecode":{"object":"0x610120806040526004361015610013575f80fd5b5f905f3560e01c908163029b464614613ee55750806304595c1a14613e915780630a6a63f114613e455780630c29202414613ac65780630cd7b88014613a8d57806315b1757014613a4057806319c75950146139e75780631a271225146139cd5780631c5ae80f1461397957806321b7cd1c146138ff57806325bbbedf146138c95780632b3129bb1461385f578063349c9179146137a957806339f51544146136b2578063431860801461351c578063442cded3146134d6578063453f4f621461349957806345c0b92d14612da5578063462dd449146110655780634903704a14612d305780634f1ef28614612a3d5780634fa27920146129eb57806352d1902d1461292b5780635353bdfd1461288e57806361a52a361461285257806367e406d5146128055780636ba4608f146127b15780636fa4469214612717578063715018a6146126595780637a1e29901461240057806389208ba9146123ac5780638a405abc1461236c5780638da5cb5b146122fb5780638fd3ab80146121a35780639f8cb3bd14612168578063a531998c14612114578063ad3cb1cc14612098578063bbae41cb14611db0578063c0e1594914611d77578063ca759f2714611d30578063dc63526614611b31578063ddea76cc14611277578063df0f32481461106a578063f178b1be14611065578063f2fde38b1461101a578063f58f952b1461047b578063f83758fe14610440578063fe4b84df1461028e5763ffa1ad741461023a575f80fd5b3461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876102736142b4565b60405191829160208352602083019061401e565b0390f35b80fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b577ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c16159067ffffffffffffffff811680159081610438575b600114908161042e575b159081610425575b506103fb578160017fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000083161784556103c6575b5061034c61583c565b61035461583c565b61035d33614fc6565b61036561583c565b6004358355610372575080f35b7fffffffffffffffffffffffffffffffffffffffffffffff00ffffffffffffffff81541690557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160018152a180f35b7fffffffffffffffffffffffffffffffffffffffffffffff00000000000000000016680100000000000000011782555f610343565b60046040517ff92ee8a9000000000000000000000000000000000000000000000000000000008152fd5b9050155f610310565b303b159150610308565b8391506102fe565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760209054604051908152f35b506104853661407a565b9291905a93828452600b60205273ffffffffffffffffffffffffffffffffffffffff92836040862054163303610f96578115610f385780855260076020526040852054804310610eda5715610e7c576104dd82614625565b9381865260076020526104f36040872054614b55565b8287526009602052604087205490600560205261051360408920546150db565b9261010084810311610e4f579193888888888d9484985b8267ffffffffffffffff8b161015610ac6576040518860208201528260408201527fffffffffffffffff0000000000000000000000000000000000000000000000008b60c01b1660608201526048815280608081011067ffffffffffffffff608083011117610a995760808101604052602081519101208b15610a6c576105b9908c83610100039106846152ac565b6105cd67ffffffffffffffff8c16876146a0565b526105e267ffffffffffffffff8b16866146a0565b506106026105fa67ffffffffffffffff8c16876146a0565b515183614503565b602081515110610a0e5760405190610619826141cf565b6020825260203681840137875b602081106109455750508660c0526020815191015160c0526020811061090f575b5081865260036020526040862061066867ffffffffffffffff8c16876146a0565b5151875260205260408620547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff81019081116108e2576106a7906150db565b610100039a6101008c116108e25760018c018c116108e2578a9b8b602061070567ffffffffffffffff6106fc816106ee6106e58d8d848a1691614f86565b86810190615087565b906101005295168a8c614f86565b359f168a6146a0565b5101516080526107148161460d565b60e05260405160a05261072b60e05160a0516141eb565b60a051508060a05152602060a05101368260051b6101005101116108de5761010051905b8260051b610100510182106108ce5750505060a051510361084a57608051969a96988b975b60a051518d10156107bd5760019061078e8e60a0516146a0565b51908c83166107ae57906107a191615cc4565b9a5b811c9c019b99610774565b6107b791615cc4565b9a6107a3565b9195995093979b91959992969a5060c051036107ec576107dc90614ec6565b989490979399959196929961052a565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f70726f6f6620646964206e6f74207665726966790000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f70726f6f66206c656e67746820646f6573206e6f74206d61746368207472656560448201527f20686569676874000000000000000000000000000000000000000000000000006064820152fd5b813581526020918201910161074f565b8980fd5b6024877f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9060200360031b1b60c0511660c0528b610647565b81518051807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101116109e1576109cc83926109c66001957fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe07fff0000000000000000000000000000000000000000000000000000000000000095016143c5565b9061582b565b51168a1a6109da828661582b565b5301610626565b60248b7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601560248201527f436964206461746120697320746f6f2073686f727400000000000000000000006044820152fd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601260045260245ffd5b7f4e487b71000000000000000000000000000000000000000000000000000000005f52604160045260245ffd5b5093959091610ad788965a90614600565b879188905b858210610e0057505060208201809211610dd35761051491828102928184041490151715610dd357610b1891610b11916143c5565b4890614792565b84875260096020526040872054907f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82168203610dd357610b83610b799493926040928a610b64614d26565b988196838d8c9552600d602052205443614600565b9360051b926154c3565b803410610d75575f8080808473ff000000000000000000000000000000000000635af1610bae614b26565b5015610d1757867f928bbf5188022bf8b9a0e59f5e81e179d0a4c729bdba2856ac971af2063fbf2b6060610c03948c9867ffffffffffffffff6040519287845216602083015260030b6040820152a234614600565b9585845260086020526040842054169081610c84575b5050507f1acf7df9f0c1b0208c23be6178950c0273f89b766805a2c0bd1e53d25c700e509183610c5b9252600d602052436040872055604051918291826140c8565b0390a28181610c675780f35b808080610c7f94335af1610c79614b26565b50614edf565b818180f35b6006602052604084205491803b15610d135784928360849260405196879586947f356de02b0000000000000000000000000000000000000000000000000000000086528c60048701526024860152604485015260648401525af18015610d0857610cf0575b8080610c19565b610cf99061419f565b610d04578385610ce9565b8380fd5b6040513d84823e3d90fd5b8480fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f4275726e206661696c65640000000000000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f496e636f72726563742066656520616d6f756e740000000000000000000000006044820152fd5b6024887f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b9092610e1a610e10858885614f86565b6020810190615087565b80915060051b90808204602014901517156109e15760400190816040116109e157600191610e47916143c5565b930190610adc565b6024897f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601660248201527f6e6f206368616c6c656e6765207363686564756c6564000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600f60248201527f7072656d61747572652070726f6f6600000000000000000000000000000000006044820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152600b60248201527f656d7074792070726f6f660000000000000000000000000000000000000000006044820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20707260448201527f6f766520706f7373657373696f6e0000000000000000000000000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611062611055614144565b61105d6157a5565b614fc6565b80f35b614167565b503461028b5761107936614266565b919061108c61108783614f44565b6142ed565b818452600c60205273ffffffffffffffffffffffffffffffffffffffff918260408620541633036111cd578493818552600b60205260408520928354858116947fffffffffffffffffffffffff00000000000000000000000000000000000000009182339116179055600c60205260408720908154169055604051943385857f686146a80f2bf4dc855942926481871515b39b508826d7982a2e0212d20552c98a80a460086020526040872054169182611144578680f35b823b156111c9578561119d819593899793889484967f4059b6d700000000000000000000000000000000000000000000000000000000865260048601526024850152336044850152608060648501526084840191614424565b03925af18015610d08576111b5575b80808080808680f35b6111be9061419f565b61028b57805f6111ac565b8680fd5b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f6e6c79207468652070726f706f7365642073746f726167652070726f76696460448201527f65722063616e20636c61696d2073746f726167652070726f766964657220726f60648201527f6c650000000000000000000000000000000000000000000000000000000000006084820152fd5b503461028b5761128636613f7c565b61129861080082969593961115614352565b6112a461108785614f44565b8215611ad357838652600b60205273ffffffffffffffffffffffffffffffffffffffff6040872054163303611a4f578386526005602052604086205494846112eb8561472b565b885b868110611569575061132e7fd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed85991604051918291602083526020830190614111565b0390a2848752600860205273ffffffffffffffffffffffffffffffffffffffff6040882054169384611366575b602087604051908152f35b843b156115655794929091879492866040519788967f545f6ec5000000000000000000000000000000000000000000000000000000008852608488019060048901528a6024890152608060448901525260a486019060a48160051b88010194809289915b83831061144b57505050505061140f8387938795937ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc87809603016064860152614424565b03925af1801561144057611428575b808080808061135b565b611432839161419f565b61143c578161141e565b5080fd5b6040513d85823e3d90fd5b929597995092977fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5c9087929597030183528735907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc18336030182121561156157828201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18085850136030182121561155d576040835284840182013603018484018201351215611559578284010180350167ffffffffffffffff8135116115595780353603602082011361155957600192826020808761153e8297968360408199015260608601908481359101614424565b94010135910152990193019301899795938c999795926113ca565b8d80fd5b8e80fd5b8c80fd5b8780fd5b909150611577818787614f86565b8035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe181360301821215611a4b5760206115b3848a8a614f86565b013591601f83166119e657821561198157660400000000000083116118f657908b9392918a8552600560205260408520918254926115f0846147a5565b90556116036115fe846143b7565b615935565b8c8560051c9188915b8183106118b057915050875260046020526040872084885260205260408720558b86526002602052604086208387526020526040862091808201357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe18284013603018112156115655767ffffffffffffffff8183850101351161156557828201810180353603602090910113611565576116a684546144b2565b601f8111611871575b508790601f818486010135116001146117925760019891818486010135611760575b5082817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9394860101358a1b9401013560031b1c19161790555b898c52600360205260408c20908c526020528060051c60408c2055888b52600660205261174060408c209160051c82546143c5565b905561174c818a6143c5565b61175682856146a0565b52019086916112ed565b8484018201016020013591507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6116d1565b848952602089209092915b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0848487010135168a1061185357600199508483018401357fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe081161061180f575b50508792010135811b01905561170b565b60207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60f886868901013560031b161c199185858801010101351690555f806117fe565b6020838601850182018101358355998a01996001909201910161179d565b8489526020808a206118a092868601850135601f810160051c830193116118a6575b601f0160051c0190615815565b5f6116af565b9091508190611893565b9260408395969798999a6118e693956118cd600180971b8c614600565b90825260046020528282209082526020522054906143c5565b92018f979695949392918e61160c565b60a484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152602160448201527f50696563652073697a65206d757374206265206c657373207468616e20325e3560648201527f30000000000000000000000000000000000000000000000000000000000000006084820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601b60448201527f53697a65206d7573742062652067726561746572207468616e203000000000006064820152fd5b608484604051907fc7b67cf3000000000000000000000000000000000000000000000000000000008252600482015260406024820152601d60448201527f53697a65206d7573742062652061206d756c7469706c65206f662033320000006064820152fd5b8a80fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20616460448201527f64207069656365730000000000000000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601b60248201527f4d75737420616464206174206c65617374206f6e6520706965636500000000006044820152fd5b503461028b57611b4036613fea565b818392935260209260058452611b5960408420546150db565b610100908103908111611d0357818452600985526040842054907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff91828101908111611cd65790611baa91846152ac565b93858501519083815260038752604081208651825287526040812054928301928311611ca9575003611bff5781611be091614462565b9182611bf3575b50506040519015158152f35b51101590505f80611be7565b60a484604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152604160248201527f6368616c6c656e676552616e6765202d312073686f756c6420616c69676e207760448201527f697468207468652076657279206c617374206c656166206f662061207069656360648201527f65000000000000000000000000000000000000000000000000000000000000006084820152fd5b807f4e487b7100000000000000000000000000000000000000000000000000000000602492526011600452fd5b6024867f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b6024847f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020611d6d600435614f44565b6040519015158152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051818152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57611de3614144565b9067ffffffffffffffff9060243582811161143c57611e06903690600401613f4e565b9390611e16610800861115614352565b67016345785d8a00009182341061203a575f8080808673ff000000000000000000000000000000000000635af1611e4b614b26565b5015610d17576001547fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000086821696611e8288614ec6565b169116176001558484526020956006875284604081205560078752846040812055600b875260408520917fffffffffffffffffffffffff00000000000000000000000000000000000000009233848254161790556008885273ffffffffffffffffffffffffffffffffffffffff6040872091168093825416179055600d875284604081205581611fb6575b50505033837f11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b962818480a33411611f46575b50604051908152f35b7ffffffffffffffffffffffffffffffffffffffffffffffffffe9cba87a2760000340190348211611f8957808080611f8394335af1610c79614b26565b5f611f3d565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b813b15610d135791849161200e93836040518096819582947f101c1eab0000000000000000000000000000000000000000000000000000000084528c6004850152336024850152606060448501526064840191614424565b03925af1801561144057908391612026575b80611f0d565b61202f9061419f565b61143c57815f612020565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f737962696c20666565206e6f74206d65740000000000000000000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576102876040516120d6816141cf565b600581527f352e302e30000000000000000000000000000000000000000000000000000000602082015260405191829160208352602083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57604060209160043561215861108782614f44565b8152600683522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040516107d08152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576121da6157a5565b7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a00805460ff8160401c1680156122e6575b6103fb577fffffffffffffffffffffffffffffffffffffffffffffff0000000000000000006002917f2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c36122ab61225f6142b4565b73ffffffffffffffffffffffffffffffffffffffff7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541660405192839260408452604084019061401e565b9060208301520390a1161790557fc7f505b2f371ae2175ee4913f4499e1f2633a7b5936321eed1cdaeb6115181d2602060405160028152a180f35b50600267ffffffffffffffff8216101561220b565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602073ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993005416604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405166040000000000008152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356123f061108782614f44565b8152600983522054604051908152f35b503461028b5761240f36614266565b9061241e610800831115614352565b67ffffffffffffffff600154168310156125fb57828452602090600b825273ffffffffffffffffffffffffffffffffffffffff92836040872054163303612577579085918583526006845260408320948386549655600b8552604084207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905560078552836040812055600d85528360408120556008855260408420541691826124f6575b505050507f14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb791604051908152a280f35b823b15610d045761254d928492836040518096819582947f2abd465c0000000000000000000000000000000000000000000000000000000084528d60048501528c6024850152606060448501526064840191614424565b03925af18015610d0857612563575b80806124c6565b61256c9061419f565b610d0457835f61255c565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602e60248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20646560448201527f6c657465206461746120736574730000000000000000000000000000000000006064820152fd5b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601960248201527f6461746120736574206964206f7574206f6620626f756e6473000000000000006044820152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576126906157a5565b5f73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c1993008054907fffffffffffffffffffffffff000000000000000000000000000000000000000082169055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e08280a380f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043561275661108782614f44565b8152600a6020526040812080549061276d8261472b565b925b82811061278c576040516020808252819061028790820187614111565b806127996001928461440f565b90549060031b1c6127aa82876146a0565b520161276f565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356127f561108782614f44565b8152600783522054604051908152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173a2aa501b19aff244d90cc15a4cf739d2725b57298152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020604051620151808152f35b503461028b5760209060207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576004359080926128d361108784614f44565b8282526005602052604082205492825b8481106128f557602086604051908152f35b818452600383526040842081855283526040842054612917575b6001016128e3565b946129236001916147a5565b95905061290f565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff7f00000000000000000000000000000000000000000000000000000000000000001630036129c15760206040517f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc8152f35b60046040517fe07c8dba000000000000000000000000000000000000000000000000000000008152fd5b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576040612a24614d26565b67ffffffffffffffff83519216825260030b6020820152f35b5060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57612a70614144565b602491823567ffffffffffffffff811161143c573660238201121561143c57806004013592612a9e8461422c565b612aab60405191826141eb565b8481526020948582019336888383010111612cf7578186928989930187378301015273ffffffffffffffffffffffffffffffffffffffff807f000000000000000000000000000000000000000000000000000000000000000016803014908115612d02575b506129c157612b1d6157a5565b821694604051907f52d1902d00000000000000000000000000000000000000000000000000000000825280826004818a5afa9182918793612cce575b5050612b8f578686604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b8590877f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc91828103612ca05750843b15612c71575080547fffffffffffffffffffffffff000000000000000000000000000000000000000016821790556040518592917fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b8480a2815115612c3b5750612c379382915190845af4612c31614b26565b91615895565b5080f35b935050505034612c49575080f35b807fb398979f0000000000000000000000000000000000000000000000000000000060049252fd5b82604051907f4c9c8ce30000000000000000000000000000000000000000000000000000000082526004820152fd5b604051907faa1d49a40000000000000000000000000000000000000000000000000000000082526004820152fd5b9080929350813d8311612cfb575b612ce681836141eb565b81010312612cf75751905f80612b59565b8580fd5b503d612cdc565b9050817f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc541614155f612b10565b503461028b57612d3f36613fea565b81835260096020526040832054917f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff83168303611d035790602093612d9d9392610b796040612d8c614d26565b9490938152600d8952205443614600565b604051908152f35b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760443567ffffffffffffffff811161143c57612df5903690600401613f4e565b90612e04610800831115614352565b6004358352600b60205273ffffffffffffffffffffffffffffffffffffffff60408420541633036134155760043583526006602052604083205415613391576004358352600d60205260408320805415613388575b50600a602052604083208054806130b5575b505060043583526006602052604083205460096020526040842055612e918354436143c5565b6024351061300b5782916004358352600760205260243560408420556006602052604083205415612fc9575b600860205273ffffffffffffffffffffffffffffffffffffffff6040842054169081612f2d575b8360043581526006602052604081205460405190602435825260208201527fc099ffec4e3e773644a4d1dda368c46af853a0eeb15babde217f53a657396e1e604060043592a280f35b600760205260408420549160066020526040852054813b15612cf757858094612f9f604051978896879586947faa27ebcc000000000000000000000000000000000000000000000000000000008652600435600487015260248601526044850152608060648501526084840191614424565b03925af18015610d0857612fb5575b8080612ee4565b612fbe9061419f565b61028b57805f612fae565b6004357f02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c28428480a2600d6020528260408120556007602052826040812055612ebd565b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f6368616c6c656e67652065706f6368206d757374206265206174206c6561737460448201527f206368616c6c656e676546696e616c6974792065706f63687320696e2074686560648201527f20667574757265000000000000000000000000000000000000000000000000006084820152fd5b6130be8161472b565b91855b8281106132e75750505090916130db611087600435614f44565b83805b8351821015613282576130f182856146a0565b5194600435875260036020526040872086885260205260408720549386916004358952600560205261312660408a20546150db565b9661010088810311613255579261313f6115fe8a6143b7565b88610100038111158061323e575b1561319b57906001613186926004358d5260046020528c8360408220915260205260408d2061317d8b8254614600565b90551b906143c5565b926131936115fe856143b7565b93909361313f565b505095509295936001926131f592956004358a52600360205260408a20818b526020525f60408b20556004358a52600260205260408a20908a5260205288604081206131e781546144b2565b80613200575b5050506143c5565b9401909192936130de565b601f808211881461321a5750505f9150555b885f806131ed565b916132375f929382865260208620940160051c8401898501615815565b5555613212565b506004358b52600560205260408b2054821061314d565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b905092919092600435855260066020526132a160408620918254614600565b90557f6e87df804629ac17804b57ba7abbdfac8bdc36bab504fb8a8801eb313a8ce7b160405160208152806132dd600435946020830190614111565b0390a25f80612e6b565b81547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808201918083116132555761331f838661440f565b919054600392831b1c613332868a6146a0565b521561335b57908291613348600195948761440f565b81939154921b1b191690558355016130c1565b60248a7f4e487b710000000000000000000000000000000000000000000000000000000081526031600452fd5b4390555f612e59565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602c60248201527f63616e206f6e6c792073746172742070726f76696e67206f6e6365206c65617660448201527f65732061726520616464656400000000000000000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603960248201527f6f6e6c79207468652073746f726167652070726f76696465722063616e206d6f60448201527f766520746f206e6578742070726f76696e6720706572696f64000000000000006064820152fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576020612d9d600435614b55565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602067ffffffffffffffff60015416604051908152f35b503461028b5760407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760043560243573ffffffffffffffffffffffffffffffffffffffff8082168092036136ae5761357d61108784614f44565b828452600b6020526040842054163381036136045781036135cd57508152600c602052604081207fffffffffffffffffffffffff0000000000000000000000000000000000000000815416905580f35b908252600c60205260408220907fffffffffffffffffffffffff000000000000000000000000000000000000000082541617905580f35b60a46040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604460248201527f4f6e6c79207468652063757272656e742073746f726167652070726f7669646560448201527f722063616e2070726f706f73652061206e65772073746f726167652070726f7660648201527f69646572000000000000000000000000000000000000000000000000000000006084820152fd5b5f80fd5b503461028b5760607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b576136f36044356024356004356147d2565b9160409491945193608085019260808652815180945260a086019360a08160051b88010194602080940192905b82821061375957888703858a015288808961374d8d61373f8c8c614111565b908482036040860152614111565b90151560608301520390f35b90919295848061379b837fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff608d60019603018652828b515191818152019061401e565b980192019201909291613720565b503461028b576137b83661407a565b909180845260056020526137cf60408520546150db565b91610100928303928311613832576137e681614625565b945b8181106137fd576040518061028788826140c8565b806138168561380f600194868a6143d2565b35866152ac565b61382082896146a0565b5261382b81886146a0565b50016137e8565b6024857f4e487b710000000000000000000000000000000000000000000000000000000081526011600452fd5b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5773ffffffffffffffffffffffffffffffffffffffff60406020926004356138b861108782614f44565b815260088452205416604051908152f35b503461028b576102876138e46138de36613fea565b90614503565b6040519182916020835251602080840152604083019061401e565b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760409060043561394161108782614f44565b8152600b60205273ffffffffffffffffffffffffffffffffffffffff8281818420541692600c60205220541682519182526020820152f35b503461028b5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760406020916004356139bd61108782614f44565b8152600583522054604051908152f35b503461028b576020611d6d6139e136613fea565b90614462565b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b5760206040517f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e8152f35b503461028b57807ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc36011261028b57602060405173fe000000000000000000000000000000000000068152f35b503461028b576040602091613aa136613fea565b90613aae61108782614f44565b82526003845282822090825283522054604051908152f35b50346136ae57613ad536613f7c565b90613ae7610800839594951115614352565b613af361108786614f44565b845f526020600b815273ffffffffffffffffffffffffffffffffffffffff908160405f2054163303613dc157865f52600a8082526107d0613b3860405f2054896143c5565b11613d3d575f5b878110613c26575050600890875f525260405f2054169182613b5f578680f35b823b156136ae57604051957fe7954aa70000000000000000000000000000000000000000000000000000000087526004870152606060248701528460648701527f07ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff85116136ae5785613bf38195935f9793608484968a9660051b809183880137850160808682030160448701520191614424565b03925af18015613c1b57613c0a5780808080808680f35b613c14915061419f565b5f806111ac565b6040513d5f823e3d90fd5b613c318189896143d2565b35895f526005845260405f20541115613cb957885f5281835260405f2090613c5a818a8a6143d2565b3582549268010000000000000000841015610a995783613c80916001809601815561440f565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff829392549160031b92831b921b191617905501613b3f565b608483604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152602c60248201527f43616e206f6e6c79207363686564756c652072656d6f76616c206f662065786960448201527f7374696e672070696563657300000000000000000000000000000000000000006064820152fd5b608482604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603a60248201527f546f6f206d616e792072656d6f76616c73207761697420666f72206e6578742060448201527f70726f76696e6720706572696f6420746f207363686564756c650000000000006064820152fd5b608490604051907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152603860248201527f4f6e6c79207468652073746f726167652070726f76696465722063616e20736360448201527f686564756c652072656d6f76616c206f662070696563657300000000000000006064820152fd5b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57602060405173ff000000000000000000000000000000000000638152f35b346136ae5760207ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae57600435613ecf61108782614f44565b5f52600d602052602060405f2054604051908152f35b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae578061080060209252f35b9181601f840112156136ae5782359167ffffffffffffffff83116136ae576020808501948460051b0101116136ae57565b9181601f840112156136ae5782359167ffffffffffffffff83116136ae57602083818601950101116136ae57565b9060607ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae576004359167ffffffffffffffff916024358381116136ae5782613fcc91600401613f1d565b939093926044359182116136ae57613fe691600401613f4e565b9091565b7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc60409101126136ae576004359060243590565b91908251928382525f5b8481106140665750507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f845f6020809697860101520116010190565b602081830181015184830182015201614028565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f1d565b60208082019080835283518092528060408094019401925f905b8382106140f157505050505090565b8451805187528301518684015294850194938201936001909101906140e2565b9081518082526020808093019301915f5b828110614130575050505090565b835185529381019392810192600101614122565b6004359073ffffffffffffffffffffffffffffffffffffffff821682036136ae57565b346136ae575f7ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc3601126136ae5760206040515f8152f35b67ffffffffffffffff8111610a9957604052565b6020810190811067ffffffffffffffff821117610a9957604052565b6040810190811067ffffffffffffffff821117610a9957604052565b90601f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0910116810190811067ffffffffffffffff821117610a9957604052565b67ffffffffffffffff8111610a9957601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01660200190565b9060407ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8301126136ae57600435916024359067ffffffffffffffff82116136ae57613fe691600401613f4e565b604051906142c1826141cf565b600582527f322e302e300000000000000000000000000000000000000000000000000000006020830152565b156142f457565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601160248201527f4461746120736574206e6f74206c6976650000000000000000000000000000006044820152fd5b1561435957565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601460248201527f4578747261206461746120746f6f206c617267650000000000000000000000006044820152fd5b9060018201809211611f8957565b91908201809211611f8957565b91908110156143e25760051b0190565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52603260045260245ffd5b80548210156143e2575f5260205f2001905f90565b601f82602094937fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe093818652868601375f8582860101520116010190565b9061446c82614f44565b918261449b575b8261447d57505090565b9091505f52600360205260405f20905f5260205260405f2054151590565b8092505f52600560205260405f2054811091614473565b90600182811c921680156144f9575b60208310146144cc57565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52602260045260245ffd5b91607f16916144c1565b6060604051614511816141b3565b5261451e61108782614f44565b5f526020906002825260405f20905f52815260405f2060405191614541836141b3565b60405180925f908054614553816144b2565b808552916001918083169081156145bf5750600114614581575b50505061457c925003826141eb565b815290565b5f90815285812095935091905b8183106145a757505061457c93508201015f808061456d565b8554878401850152948501948694509183019161458e565b91505061457c9593507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f808061456d565b91908203918211611f8957565b67ffffffffffffffff8111610a995760051b60200190565b9061462f8261460d565b60409061463f60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe061466d829561460d565b01915f5b83811061467e5750505050565b602090825161468c816141cf565b5f8152825f81830152828601015201614671565b80518210156143e25760209160051b010190565b906146be8261460d565b6040906146ce60405191826141eb565b8381527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe06146fc829561460d565b01915f5b83811061470d5750505050565b602090825161471b816141b3565b6060815282828601015201614700565b906147358261460d565b61474260405191826141eb565b8281527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0614770829461460d565b0190602036910137565b906701518000000000009180830292830403611f8957565b81810292918115918404141715611f8957565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8114611f895760010190565b915f6147e061108785614f44565b8115614ac857835f52600560205260405f2054926147fd836146b4565b926148078161472b565b946148118261472b565b965f905f945f5b848110614905575b505050505081155f1461486d5750505050505060405161483f816141b3565b5f81526040519161484f836141b3565b5f83526040519161485f836141b3565b5f83525f3681379291905f90565b81969293949596105f146148ff57614884816146b4565b9561488e8261472b565b956148988361472b565b955f5b8481106148a9575050505050565b806148b6600192846146a0565b516148c1828d6146a0565b526148cc818c6146a0565b506148d781856146a0565b516148e2828c6146a0565b526148ed81866146a0565b516148f8828b6146a0565b520161489b565b50919391565b825f52600360205260405f20815f5260205260405f2054614929575b600101614818565b92958187101580614abf575b15614a8e57825f52600260205260405f20845f5260205260405f206040519061495d826141b3565b60405190815f82549261496f846144b2565b8084529360018116908115614a4e5750600114614a0c575b50614994925003826141eb565b81526149a0828b6146a0565b526149ab818a6146a0565b50836149b7828c6146a0565b52825f52600360205260405f20845f5260205260405f2054908160051b9180830460201490151715611f89576149fe816001936149f88f94614a04956146a0565b526147a5565b976147a5565b939050614921565b9150505f528160205f20915f925b818410614a325750506020614994928201015f614987565b6020919250806001915483868801015201920191908391614a1a565b602093506149949592507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0091501682840152151560051b8201015f614987565b95614a9986836143c5565b811015614aab57614a046001916147a5565b505050505091506001915f80808080614820565b50858110614935565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601c60248201527f4c696d6974206d7573742062652067726561746572207468616e2030000000006044820152fd5b3d15614b50573d90614b378261422c565b91614b4560405193846141eb565b82523d5f602084013e565b606090565b5f80916040516020810191825260208152614b6f816141cf565b519073fe000000000000000000000000000000000000065afa614b90614b26565b9015614baa576020818051810103126136ae576020015190565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f52616e646f6d6e65737320707265636f6d70696c652063616c6c206661696c6560448201527f64000000000000000000000000000000000000000000000000000000000000006064820152fd5b908160809103126136ae576040519067ffffffffffffffff6080830181811184821017610a995760405281518060070b81036136ae578352602082015190811681036136ae5760208301526040810151908160030b82036136ae5760609160408401520151606082015290565b15614ca257565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f6661696c656420746f2076616c69646174653a207072696365206d757374206260448201527f652067726561746572207468616e2030000000000000000000000000000000006064820152fd5b6040908151917fa4ae35e00000000000000000000000000000000000000000000000000000000083527f150ac9b959aee0051e4091f0ef5216d941f590e1c5e7f91cf7635b5c11628c0e9283600482015262015180602482015260809373a2aa501b19aff244d90cc15a4cf739d2725b5729918581604481865afa5f9181614ea7575b50614e92575084907f32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120614df0614ddd614b26565b865191829160208352602083019061401e565b0390a160248451809481937f96834ad300000000000000000000000000000000000000000000000000000000835260048301525afa938415614e88575f94614e59575b5050614e445f845160070b13614c9b565b825167ffffffffffffffff1692015160030b90565b614e79929450803d10614e81575b614e7181836141eb565b810190614c2e565b915f80614e33565b503d614e67565b82513d5f823e3d90fd5b809550614e4492505f91505160070b13614c9b565b614ebf919250873d8911614e8157614e7181836141eb565b905f614da9565b67ffffffffffffffff809116908114611f895760010190565b15614ee657565b60646040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601060248201527f5472616e73666572206661696c65642e000000000000000000000000000000006044820152fd5b67ffffffffffffffff6001541681109081614f5d575090565b90505f52600b60205273ffffffffffffffffffffffffffffffffffffffff60405f205416151590565b91908110156143e25760051b810135907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc1813603018212156136ae570190565b73ffffffffffffffffffffffffffffffffffffffff809116908115615057577f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300805490837fffffffffffffffffffffffff00000000000000000000000000000000000000008316179055167f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e05f80a3565b60246040517f1e4fbdf70000000000000000000000000000000000000000000000000000000081525f6004820152fd5b9035907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe1813603018212156136ae570180359067ffffffffffffffff82116136ae57602001918160051b360383136136ae57565b610100908060801c806152a0575b508060401c8061526d575b508060201c8061523a575b508060101c80615207575b508060081c806151d4575b508060041c806151a1575b508060021c8061516e575b508060011c6151405761513d91614600565b90565b507ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe810191508111611f8957905f61512b565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc810191508111611f8957905f615120565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8810191508111611f8957905f615115565b917ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0810191508111611f8957905f61510a565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0810191508111611f8957905f6150ff565b917fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc0810191508111611f8957905f6150f4565b9150506080905f6150e9565b916040918251916152bc836141cf565b5f83525f602080940152845f5260068352835f2054821015615452576001947fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff86831b818101908111611f895796905f9391825b61536e575050505f5260048352835f20855f528352615332845f2054826143c5565b82811115615358575061534491614600565b915192615350846141cf565b835282015290565b905060018501809511611f895761534491614600565b909197835f5260058752875f205481101561543b57835f52600494858852885f20825f528852866153a28a5f2054836143c5565b1161541b576153c390855f52868952895f20835f528952895f2054906143c5565b94838a01908a82116153ef575090826153dd921b906143c5565b975b8015611f89578201919082615310565b6011907f4e487b71000000000000000000000000000000000000000000000000000000005f525260245ffd5b94838a01908a82116153ef57509082615435921b90614600565b976153df565b82890190898211611f895782615435921b90614600565b6064838551907f08c379a00000000000000000000000000000000000000000000000000000000082526004820152601860248201527f4c65616620696e646578206f7574206f6620626f756e647300000000000000006044820152fd5b63ffffffff16604d8111611f8957600a0a90565b93929190841580159061579d575b156157195767ffffffffffffffff1690811561569557821561561157600381900b5f81126155a3575061551561550961551b9361477a565b9163ffffffff166154af565b90614792565b8015610a6c5761553d9261553891671bc16d674ec8000004614792565b614792565b9060058202821590838104600514821715611f895760649004908360021b848104600414821715611f89578284106155785750505050505f90565b60649004831061558d575061513d9250614600565b9150508180046001141715611f89576064900490565b90507fffffffffffffffffffffffffffffffffffffffffffffffffffffffff800000008114611f89576155dd905f0363ffffffff166154af565b90671bc16d674ec800009180830292830403611f89576155fc9061477a565b928315610a6c5761553d936155389204614792565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603360248201527f6661696c656420746f2076616c69646174653a207261772073697a65206d757360448201527f742062652067726561746572207468616e2030000000000000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603860248201527f6661696c656420746f2076616c69646174653a204174746f46494c207072696360448201527f65206d7573742062652067726561746572207468616e203000000000000000006064820152fd5b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603c60248201527f6661696c656420746f2076616c69646174653a20657374696d6174656420676160448201527f7320666565206d7573742062652067726561746572207468616e2030000000006064820152fd5b5048156154d1565b73ffffffffffffffffffffffffffffffffffffffff7f9016d09d72d40fdae2fd8ceac6b6234c7706214fd39c1cd1e609a0528c199300541633036157e557565b60246040517f118cdaa7000000000000000000000000000000000000000000000000000000008152336004820152fd5b818110615820575050565b5f8155600101615815565b9081518110156143e2570160200190565b60ff7ff0c57e16840df040f15088dc2f81fe391c3923bec73e23a9662efc9c229c6a005460401c161561586b57565b60046040517fd7e6bcf8000000000000000000000000000000000000000000000000000000008152fd5b906158d457508051156158aa57805190602001fd5b60046040517fd6bda275000000000000000000000000000000000000000000000000000000008152fd5b8151158061592c575b6158e5575090565b60249073ffffffffffffffffffffffffffffffffffffffff604051917f9996b315000000000000000000000000000000000000000000000000000000008352166004820152fd5b50803b156158dd565b7f7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8111615c4057610100907f80000000000000000000000000000000000000000000000000000000000000008114611f8957805f031680615c37575b6fffffffffffffffffffffffffffffffff8116615c06575b77ffffffffffffffff0000000000000000ffffffffffffffff8116615bd5575b7bffffffff00000000ffffffff00000000ffffffff00000000ffffffff8116615ba4575b7dffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff8116615b73575b7eff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff00ff8116615b42575b7f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f8116615b11575b7f33333333333333333333333333333333333333333333333333333333333333338116615ae0575b7f555555555555555555555555555555555555555555555555555555555555555516615ab35790565b7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8101908111611f895790565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe8101908111611f895790615a8a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc8101908111611f895790615a62565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff88101908111611f895790615a3a565b907ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff08101908111611f895790615a13565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08101908111611f8957906159ed565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc08101908111611f8957906159c9565b907fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff808101908111611f8957906159a9565b60ff9150615991565b60846040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f496e7075742065786365656473206d6178696d756d20696e743235362076616c60448201527f75650000000000000000000000000000000000000000000000000000000000006064820152fd5b5f5260205260205f60408160025afa156136ae577fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3f5f51169056fea2646970667358221220a2d6c54c44a68038f7d59aa211cb3b39a5f231a0e7b9f3f652141521dc58b6fd64736f6c63430008170033","sourceMap":"2048:37005:43:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2267:2;2048:37005;2267:2;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;8837:64:24;2048:37005:43;;;;;;;4301:16:24;2048:37005:43;;;;4726:16:24;;:34;;;;2048:37005:43;;4790:16:24;:50;;;;2048:37005:43;4855:13:24;:30;;;;2048:37005:43;4851:91:24;;;2048:37005:43;;;;;;;;4979:67:24;;2048:37005:43;6893:76:24;;;:::i;:::-;;;:::i;:::-;6961:1;7731:10:43;6961:1:24;:::i;:::-;6893:76;;:::i;:::-;2048:37005:43;;;;5066:101:24;;2048:37005:43;;;5066:101:24;2048:37005:43;;;;;;5142:14:24;2048:37005:43;;;;;;5142:14:24;2048:37005:43;;4979:67:24;2048:37005:43;;;;;;4979:67:24;;;4851:91;2048:37005:43;;;4908:23:24;;;;4855:30;4872:13;;;4855:30;;;4790:50;4818:4;4810:25;:30;;-1:-1:-1;4790:50:24;;4726:34;;;-1:-1:-1;4726:34:24;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;23384:9;;;;2048:37005;;;;23466:15;2048:37005;;;;;;;;;;23452:10;:36;2048:37005;;23557:11;;2048:37005;;;;;23633:18;2048:37005;;;;;;23680:12;;:30;2048:37005;;23752:40;2048:37005;;23889:47;;;:::i;:::-;2048:37005;;;;23633:18;2048:37005;;29356:40;2048:37005;;;;29356:40;:::i;:::-;2048:37005;;;24030:14;2048:37005;;;;;;;24103:11;2048:37005;;24092:30;2048:37005;;;;24092:30;:::i;:::-;2048:37005;24086:3;2048:37005;;;;;;24141:12;;;;;;;;;24136:1885;24168:3;2048:37005;;;;24155:11;;;;2048:37005;;25323:32;2048:37005;25323:32;;2048:37005;;;;;;;;;;;;;;;25323:32;;;2048:37005;;;;;;;;;;;;;;;;;;;;;25323:32;;25404:18;2048:37005;;;;25570:47;2048:37005;;;24086:3;2048:37005;;;25570:47;;:::i;:::-;25554:63;2048:37005;;;25554:63;;:::i;:::-;;;2048:37005;;;25554:63;;:::i;:::-;;25674:41;25693:13;2048:37005;;;25693:13;;:::i;:::-;;2048:37005;25674:41;;:::i;:::-;2048:37005;370:8:40;;2048:37005:43;370:21:40;2048:37005:43;;;;;;;;:::i;:::-;;;;;;;;;;480:10:40;492:6;2048:37005:43;492:6:40;;;;595:18;;;;;2048:37005:43;;;;;;;;;;;;;475:104:40;2048:37005:43;;;;25773:15;2048:37005;;;;;25796:13;2048:37005;;;25796:13;;:::i;:::-;;2048:37005;;;;;;;;;;;;;;;;;25762:61;;;:::i;:::-;24086:3;2048:37005;;24086:3;2048:37005;;;;25821:1;2048:37005;;;;;;;;;;25919:13;2048:37005;25903:9;2048:37005;25875:15;:9;2048:37005;;;;;25875:9;;:::i;:::-;:15;;;;;:::i;:::-;;;;2048:37005;;25903:9;;;:::i;:::-;2048:37005;;;25919:13;;:::i;:::-;;:20;2048:37005;;;;;;:::i;:::-;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;24103:11;2048:37005;;;;;;;;;;;;24103:11;2048:37005;;;;;;;;1260:12:44;;;;;2048:37005:43;1260:30:44;2048:37005:43;;1351:50:44;;1878:13;;1351:50;;1878:13;;1911:3;1897:12;;2048:37005:43;1893:16:44;;;;;25821:1:43;2042:8:44;;;;;;:::i;:::-;2048:37005:43;;;;;25821:1;;10374:22:44;;;;:::i;:::-;2064:207;;1025:5:42;;1911:3:44;2048:37005:43;1878:13:44;;;;2064:207;10374:22;;;:::i;:::-;2064:207;;;1893:16;;;;;;;;;;;;;;;1351:58;;;2048:37005:43;;24168:3;;;:::i;:::-;24141:12;;;;;;;;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;25773:15;2048:37005;;;;;;;;;;500:3:40;534:8;;2048:37005:43;;;25323:32;2048:37005;;;;;534:34:40;2048:37005:43;;543:24:40;25821:1:43;2048:37005;25323:32;2048:37005;;;543:24:40;:::i;:::-;534:34;;:::i;:::-;2048:37005:43;;519:49:40;;;;;;:::i;:::-;;2048:37005:43;480:10:40;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;24155:11;;;;;;26425:22;24155:11;;26438:9;26425:22;;:::i;:::-;28560:24;28599:13;;28594:217;28614:17;;;;;;2048:37005;;;;;;;;;;26491:4;2048:37005;;;;;;;;;;;;;;;27871:23;26424:72;;;;:::i;:::-;27881:13;27871:23;;:::i;:::-;2048:37005;;;24030:14;2048:37005;;;;;;;;;;;;;;28050:203;28199:44;28004:16;;;2048:37005;28004:16;;;;:::i;:::-;2048:37005;;;;;;;;28214:22;2048:37005;;;;23680:12;28199:44;:::i;:::-;2048:37005;24103:11;2048:37005;28050:203;;:::i;:::-;8244:9;;:19;2048:37005;;;8317:34;;;;2183:42;8317:34;;;;:::i;:::-;;2048:37005;;;;28295:59;2048:37005;28372:20;2048:37005;;;;;;;;;;;;;;;25773:15;2048:37005;;;;;28295:59;8244:9;28372:20;:::i;:::-;2048:37005;;;;26611:15;2048:37005;;;;;;;26651:26;;26647:160;;28594:217;2048:37005;;;26886:35;2048:37005;;26886:35;2048:37005;;28214:22;2048:37005;;23680:12;2048:37005;;;;;;26886:35;;;;;:::i;:::-;;;;27131:10;;27127:144;;2048:37005;;27127:144;23452:10;;;27224:36;23452:10;;27176:34;;;;:::i;:::-;;27224:36;:::i;:::-;27127:144;;2048:37005;;26647:160;26747:16;2048:37005;;;;;;26697:95;;;;;;2048:37005;;;;;;;26697:95;;;;;2048:37005;26697:95;;;2048:37005;26697:95;;2048:37005;;;;;;;;;;;;;26697:95;;;;;;;;26647:160;;;;;26697:95;;;;:::i;:::-;2048:37005;;26697:95;;;;2048:37005;;;;26697:95;2048:37005;;;;;;;;;26697:95;2048:37005;;;;;;;;;;;;;;;23466:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;28633:3;28772:9;;:15;:9;;;;;:::i;:::-;2048:37005;28772:15;;;;:::i;:::-;2048:37005;;;24103:11;2048:37005;;;;;;;;;;;;;;;;;;;;;25821:1;28750:50;;;;:::i;:::-;28633:3;2048:37005;28599:13;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;23466:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2357:1:23;2048:37005:43;;:::i;:::-;2303:62:23;;:::i;:::-;2357:1;:::i;:::-;2048:37005:43;;;;:::i;:::-;;;;;;;;:::i;:::-;16941:18;;16933:48;16941:18;;;:::i;:::-;16933:48;:::i;:::-;2048:37005;;;16999:30;2048:37005;;;;;;;;;;17040:10;16999:51;2048:37005;;;;;;;17160:15;2048:37005;;;;;;;;;;;;;17040:10;;;2048:37005;;;;;16999:30;2048:37005;;;;;;;;;;;;;17040:10;;17296:61;;;;;;17390:15;2048:37005;;;;;;;17426:26;;17422:155;;2048:37005;;;17422:155;17468:98;;;;;;2048:37005;17468:98;;;;;;;;;;2048:37005;17468:98;;2048:37005;17468:98;;2048:37005;;;;;17040:10;2048:37005;;;;;;;;;;;;;;:::i;:::-;17468:98;;;;;;;;;;17422:155;;;;;;2048:37005;;;17468:98;;;;:::i;:::-;2048:37005;;17468:98;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;20193:72;2529:4;20201:39;;;;;;;20193:72;:::i;:::-;20275:48;20283:18;;;:::i;20275:48::-;20341:11;;2048:37005;;;;;20402:15;2048:37005;;;;;;;;20428:10;20402:36;2048:37005;;;;;20514:11;2048:37005;;;;;;20570:31;;;;;:::i;:::-;20618:13;20633:11;;;;;;2048:37005;;20795:28;2048:37005;;;;;;;;;;;;;;:::i;:::-;20795:28;;;2048:37005;;;20857:15;2048:37005;;;;;;;;20893:26;;20889:135;;20613:168;2048:37005;;;;;;;;20889:135;20935:78;;;;;2048:37005;;;;;;;;;;20935:78;;;2048:37005;20935:78;;2048:37005;;;20935:78;2048:37005;20935:78;;2048:37005;;;;;;;;;;;;;;;;;;20514:11;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;20935:78;;;;;;;;;;2048:37005;20889:135;;;;;;;20935:78;;;;;:::i;:::-;2048:37005;;20935:78;;;2048:37005;;;;20935:78;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;20935:78;2048:37005;;;20646:3;20687:12;;;;;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;20707:12;;;;;:::i;:::-;:20;2048:37005;;;;;21246:116;;21375:12;;21371:102;;2316:7;21486:24;;21482:120;;2048:37005;;;;;;;;20514:11;2048:37005;;;;;;;;21679:20;;;;:::i;:::-;2048:37005;;38022:21;38033:9;;;:::i;:::-;38022:21;:::i;:::-;1025:5:42;;20514:11:43;1025:5:42;35062:13:43;;35057:129;35077:5;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;21756:9;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;35057:129;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;1025:5:42;20514:11:43;1025:5:42;2048:37005:43;;;;;;;21852:16;2048:37005;;21852:36;2048:37005;;;1025:5:42;20514:11:43;1025:5:42;2048:37005:43;;21852:36;:::i;:::-;2048:37005;;20756:14;;;;:::i;:::-;20742:28;;;;:::i;:::-;2048:37005;;20618:13;;;;;2048:37005;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;21799:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;20514:11;2048:37005;;;;-1:-1:-1;2048:37005:43;;;;;20514:11;2048:37005;;;;:::i;:::-;;;;;;;-1:-1:-1;2048:37005:43;;;;35084:3;2048:37005;;;;;;;;;35145:30;2048:37005;;35115:16;2048:37005;;;;35115:16;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;35145:30;;:::i;:::-;35084:3;2048:37005;35062:13;;;;;;;;;;;21482:120;2048:37005;;;;21533:58;;;;2048:37005;21533:58;;2048:37005;;;;;;;;;;;;;;;;;;;;;21533:58;21371:102;2048:37005;;;;21410:52;;;;2048:37005;21410:52;;2048:37005;;;;;;;;;;;;;;;;21410:52;21246:116;2048:37005;;;;21297:54;;;;2048:37005;21297:54;;2048:37005;;;;;;;;;;;;;;;;21297:54;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;9446:11;2048:37005;;9435:30;2048:37005;;;;9435:30;:::i;:::-;9429:3;2048:37005;;;;;;;;;;;9537:14;2048:37005;;;;;;;;;;;;;;;;;9515:51;;;;;:::i;:::-;9584:10;;;;2048:37005;;;;;9598:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;9584:53;;2048:37005;;9724:25;;;;:::i;:::-;:51;;;;2048:37005;;;;;;;;;;;9724:51;2048:37005;-1:-1:-1;9753:22:43;;-1:-1:-1;9724:51:43;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;2267:2;2048:37005;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;:::i;:::-;17890:39;;17882:72;2529:4;17890:39;;;17882:72;:::i;:::-;1025:5:42;18019:9:43;;;:21;2048:37005;;;8317:34;;;;2183:42;8317:34;;;;:::i;:::-;;2048:37005;;;235:1:42;2048:37005:43;;;;;18116:15;;;;:::i;:::-;2048:37005;;;;235:1:42;2048:37005:43;;;;;;18141:16;2048:37005;;;;;;;18178:18;2048:37005;;;;;;;18289:15;2048:37005;;;;;;;18314:10;;2048:37005;;;;;;;18334:15;2048:37005;;;;;;;;;;;;;;;;18381:22;2048:37005;;;;;;;18443:26;18439:127;;2048:37005;18314:10;;;;18580:33;;;;;18019:9;18703:20;18699:168;;2048:37005;;;;;;;;18699:168;2048:37005;18019:9;2048:37005;18019:9;;2048:37005;;;;18314:10;;;18820:36;18314:10;;18758:48;;;;:::i;18820:36::-;18699:168;;;2048:37005;;;;;;;;;;18439:127;18485:70;;;;;2048:37005;;;;;;;;18485:70;;;;;;2048:37005;18485:70;;;2048:37005;18485:70;;2048:37005;18314:10;2048:37005;;;;;;;;;;;;;;:::i;:::-;18485:70;;;;;;;;;;;;;18439:127;;;;18485:70;;;;:::i;:::-;2048:37005;;18485:70;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;9916:48;9924:18;;;:::i;9916:48::-;2048:37005;;9981:16;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2377:4;2048:37005;;;;;;;;;;;;;;;2303:62:23;;:::i;:::-;8837:64:24;2048:37005:43;;;;;;;6431:44:24;;;;2048:37005:43;6427:105:24;;2048:37005:43;8004:1;2048:37005;8022:59;2048:37005;;;:::i;:::-;;811:66:30;2048:37005:43;;;;;;;;;;;;;;;:::i;:::-;;;;;;8022:59;;;2048:37005;;;;6656:20:24;2048:37005:43;;;8004:1;2048:37005;;6656:20:24;2048:37005:43;;6431:44:24;2048:37005:43;8004:1;2048:37005;;;6450:25:24;;6431:44;;2048:37005:43;;;;;;;;;;;;;;1280:65:23;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;2316:7;2048:37005;;;;;;;;;;;;;;;;;;;;11982:48;11990:18;;;:::i;11982:48::-;2048:37005;;12047:14;2048:37005;;;;;;;;;;;;;;;;;;:::i;:::-;19057:39;19049:72;2529:4;19057:39;;;19049:72;:::i;:::-;2048:37005;19144:13;2048:37005;;19135:22;;;19131:88;;2048:37005;;;;;19237:15;2048:37005;;;;;;;;;;19263:10;19237:36;2048:37005;;;;;;;;19361:16;2048:37005;;;;;;;;;;;19237:15;2048:37005;;;;;;;;;;;19476:18;2048:37005;;;;;;;19515:22;2048:37005;;;;;;;19596:15;2048:37005;;;;;;;19632:26;;19628:133;;2048:37005;;;;;19775:39;2048:37005;;;;;;19775:39;2048:37005;;19628:133;19674:76;;;;;2048:37005;;;;;;;19674:76;;;;;;2048:37005;19674:76;;;2048:37005;19674:76;;2048:37005;;;;;;;;;;;;;;;;:::i;:::-;19674:76;;;;;;;;;;19628:133;;;;;19674:76;;;;:::i;:::-;2048:37005;;19674:76;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;19131:88;2048:37005;;;19173:35;;;2048:37005;;19173:35;;2048:37005;;;;;;;;;;;19173:35;2048:37005;;;;;;;;;;;;2303:62:23;;:::i;:::-;2048:37005:43;;1280:65:23;2048:37005:43;;;;;;;;;3975:40:23;;;;2048:37005:43;;;;;;;;;;;;;;;;12280:48;12288:18;;;:::i;12280:48::-;2048:37005;;12367:17;2048:37005;;;;;;;12427:30;;;;:::i;:::-;12472:13;12487:19;;;;;;2048:37005;;;;;;;;;;;;;;:::i;12508:3::-;12539:11;;2048:37005;12539:11;;;:::i;:::-;2048:37005;;;;;;12527:23;;;;:::i;:::-;2048:37005;;12472:13;;2048:37005;;;;;;;;;;;;;;;;;10381:48;10389:18;;;:::i;10381:48::-;2048:37005;;10446:18;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2626:42;2048:37005;;;;;;;;;;;;;;;;;;2580:5;2048:37005;;;;;;;;;;;;;;;;;;;;12871:19;12910:18;12902:48;12910:18;;;:::i;12902:48::-;2048:37005;;;12982:11;2048:37005;;;;;;13015:13;;13030:14;;;;;;2048:37005;;;;;;;;13046:3;2048:37005;;;13069:15;2048:37005;;;;;;;;;;;;;;13065:81;;13046:3;2048:37005;;13015:13;;13065:81;13118:13;;2048:37005;13118:13;;:::i;:::-;13065:81;;;;;2048:37005;;;;;;;;;;;;;5115:6:25;2048:37005:43;5106:4:25;5098:23;5094:145;;2048:37005:43;;;811:66:30;2048:37005:43;;;5094:145:25;2048:37005:43;;;5199:29:25;;;;2048:37005:43;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4692:6:25;;2048:37005:43;4683:4:25;;4675:23;:120;;;;;2048:37005:43;4658:251:25;;;2303:62:23;;:::i;:::-;2048:37005:43;;;;;6156:52:25;2048:37005:43;6156:52:25;;;;2048:37005:43;6156:52:25;;;;;;;;;;;2048:37005:43;-1:-1:-1;;6152:437:25;;2048:37005:43;;;;6518:60:25;;;;2048:37005:43;6518:60:25;;2048:37005:43;6518:60:25;6152:437;811:66:30;;;;6250:40:25;;;;6246:120;;1748:29:30;;;:34;1744:119;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;2407:36:30;2048:37005:43;;2407:36:30;2048:37005:43;;2458:15:30;:11;;4049:25:33;4091:55;4049:25;;;;;;;;;;:::i;:::-;4091:55;;:::i;:::-;;2048:37005:43;;2454:148:30;6163:9;;;;;;6159:70;;2454:148;2048:37005:43;;6159:70:30;6199:19;;2048:37005:43;6199:19:30;;;1744:119;2048:37005:43;;;1805:47:30;;;;2048:37005:43;1805:47:30;;2048:37005:43;1805:47:30;6246:120:25;2048:37005:43;;6317:34:25;;;;2048:37005:43;6317:34:25;;2048:37005:43;6317:34:25;6156:52;;;;;;;;;;;;;;;;;:::i;:::-;;;2048:37005:43;;;;;6156:52:25;;;;;2048:37005:43;;;;6156:52:25;;;;;4675:120;2048:37005:43;;;811:66:30;2048:37005:43;;4753:42:25;;4675:120;;;2048:37005:43;;;;;;;;:::i;:::-;;;;27408:14;2048:37005;;;;;;;;;;;;;;27485:16;2048:37005;27485:16;27519:203;27485:16;;27668:44;2048:37005;27485:16;;:::i;:::-;2048:37005;;;;;27683:22;2048:37005;;;;27668:12;:44;:::i;27519:203::-;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;30423:39;30415:72;2529:4;30423:39;;;30415:72;:::i;:::-;2048:37005;;;;30519:15;2048:37005;;;;;;;;30505:10;:36;2048:37005;;;;;;30621:16;2048:37005;;;;;;30621:27;2048:37005;;;;;;30712:22;2048:37005;;;;;;;30712:48;30708:123;;2048:37005;;30920:17;2048:37005;;;;;;;31003:13;30999:387;;2048:37005;;;;;;;30621:16;2048:37005;;;;;;31443:14;2048:37005;;;;;;31521:32;2048:37005;;31521:12;:32;:::i;:::-;2048:37005;;31504:49;31500:161;;2048:37005;;;;;;31670:18;2048:37005;;;;;;;;30621:16;2048:37005;;;;;;31876:28;31872:208;;2048:37005;32113:15;2048:37005;;;;;;;;32149:26;;32145:170;;2048:37005;;;;;;30621:16;2048:37005;;;;;;;;;;;;;;;;;32329:65;2048:37005;;;32329:65;;2048:37005;;32145:170;31670:18;2048:37005;;;;;;;30621:16;2048:37005;;;;;;32191:113;;;;;2048:37005;;;;;;32191:113;;;;;;;2048:37005;32191:113;;2048:37005;;;32191:113;;2048:37005;;;;;;;;;;;;;;;;;;;:::i;:::-;32191:113;;;;;;;;;;32145:170;;;;;32191:113;;;;:::i;:::-;2048:37005;;32191:113;;;;31872:208;2048:37005;;31925:19;;;;30712:22;2048:37005;;;;;;;31670:18;2048:37005;;;;;;;31872:208;;31500:161;2048:37005;;;31569:81;;;2048:37005;;31569:81;;2048:37005;;;;;;;;;;;;;;;;;;;;;31569:81;30999:387;31069:24;;;:::i;:::-;31113:13;;31128;;;;;;2048:37005;;;;;32537:48;32545:18;2048:37005;;32545:18;:::i;32537:48::-;32595:22;;32668:3;2048:37005;;32647:19;;;;;32722:11;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;;;;;;33152:5;;2048:37005;;;;;35407:11;2048:37005;;35396:30;2048:37005;;;;35396:30;:::i;:::-;2048:37005;35390:3;2048:37005;;;;;;38033:9;38022:21;38033:9;;;:::i;38022:21::-;2048:37005;35390:3;2048:37005;35644:8;;;:38;;;35637:177;35644:38;;;2048:37005;;35748:15;2048:37005;;;;;;;;;;;;;;;;;;;;35698:36;2048:37005;;;35698:36;:::i;:::-;2048:37005;;;35748:15;;:::i;:::-;38033:9;38022:21;38033:9;;;:::i;38022:21::-;35637:177;;;;;35644:38;;;;;;;;2048:37005;35644:38;32686:48;35644:38;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;33223:9;2048:37005;;;;;;;;;;;;;;;;;;:::i;:::-;;;;35637:177;32686:48;;;;:::i;:::-;32668:3;2048:37005;32632:13;;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;35407:11;2048:37005;;;;;;;:::i;:::-;;;;;35644:38;2048:37005;;;;;35407:11;2048:37005;;;;;;35656:26;;35644:38;;2048:37005;;;;;;;;;;32647:19;;;;;;;2048:37005;;;;30621:16;2048:37005;;32754:37;2048:37005;;;;;;32754:37;:::i;:::-;2048:37005;;31336:39;2048:37005;;;;;;;;;;;;;;;:::i;:::-;31336:39;;;30999:387;;;;31143:3;2048:37005;;;;;;;;;;;;31189:29;;;;:::i;:::-;2048:37005;;;;;;;;31166:52;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;31113:13;;2048:37005;;;;;;;;;;30708:123;30808:12;2048:37005;;30708:123;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;8676:13;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;16262:48;16270:18;;;:::i;16262:48::-;2048:37005;;;16353:15;2048:37005;;;;;;;16419:10;16393:36;;2048:37005;;16516:44;;2048:37005;;;;;16689:30;2048:37005;;;;;;;;;;;;;16512:314;2048:37005;;;16757:30;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;37518:11;2048:37005;;37507:30;2048:37005;;;;37507:30;:::i;:::-;37501:3;;2048:37005;;;;;;;;37592:51;;;:::i;:::-;37658:13;37673:21;;;;;;2048:37005;;;;;;;:::i;37696:3::-;37749:13;37727:41;37749:13;;2048:37005;37749:13;;;;:::i;:::-;2048:37005;37727:41;;:::i;:::-;37715:53;;;;:::i;:::-;;;;;;:::i;:::-;;2048:37005;37658:13;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;10618:48;10626:18;;;:::i;10618:48::-;2048:37005;;10683:15;2048:37005;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;10908:48;10916:18;;;:::i;10908:48::-;2048:37005;;10974:15;2048:37005;;;;;;;;;;;10998:30;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;10144:48;10152:18;;;:::i;10144:48::-;2048:37005;;10209:11;2048:37005;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;:::i;:::-;;;;;;;;;;;;;;;2779:66;2048:37005;;;;;;;;;;;;;;;;;;2435:42;2048:37005;;;;;;;;;;;;;;:::i;:::-;11690:18;11682:48;11690:18;;;:::i;11682:48::-;2048:37005;;11747:15;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;22230:39;22222:72;2529:4;22230:39;;;;;;22222:72;:::i;:::-;22304:48;22312:18;;;:::i;22304:48::-;2048:37005;;;;22370:15;2048:37005;;;;;;;;;;22396:10;22370:36;2048:37005;;;;;22503:17;;2048:37005;;2377:4;22485:49;2048:37005;;;;22485:49;;:::i;:::-;:74;2048:37005;;;22653:19;;;;;;2048:37005;;22881:15;2048:37005;;;;;;;;;;22917:26;;22913:132;;2048:37005;;;22913:132;22959:75;;;;;2048:37005;;22959:75;2048:37005;22959:75;;2048:37005;22959:75;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;22714:11;2048:37005;;;;;;;;;;;;;;;;;;;;;:::i;:::-;22959:75;;;;;;;;;;22913:132;;;;;2048:37005;;;22959:75;;;;;:::i;:::-;2048:37005;22959:75;;;;2048:37005;;;;;;;;;22674:3;22700:11;;;;;:::i;:::-;2048:37005;;;;22714:11;2048:37005;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;22825:11;;;;;;:::i;:::-;2048:37005;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;22638:13;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2183:42;2048:37005;;;;;;;;;;;;;;;;11139:48;11147:18;;;:::i;11139:48::-;2048:37005;;11204:22;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;2529:4;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;:::i;:::-;;;:::o;:::-;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;31216:1;2048:37005;;;;;;;:::o;:::-;;;;;;;;;;:::o;:::-;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;:::o;:::-;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;:::o;9027:191::-;;9122:18;;;:::i;:::-;:50;;;;9027:191;9122:89;;;9115:96;;9027:191;:::o;9122:89::-;2048:37005;;;-1:-1:-1;2048:37005:43;9176:15;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;9176:35;;9027:191;:::o;9122:50::-;2048:37005;;;-1:-1:-1;2048:37005:43;9154:11;2048:37005;;;-1:-1:-1;2048:37005:43;;9144:28;;9122:50;;;2048:37005;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;11309:198;2048:37005;;;;;;:::i;:::-;;11410:48;11418:18;;;:::i;11410:48::-;-1:-1:-1;2048:37005:43;;;11475:9;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;;;;;;:::i;:::-;;;;;-1:-1:-1;2048:37005:43;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;11309:198;:::o;2048:37005::-;-1:-1:-1;2048:37005:43;;;;;;;;-1:-1:-1;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;:::o;:::-;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;-1:-1:-1;2048:37005:43;;;;;;;;;;:::o;:::-;;;;;;;;:::i;:::-;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;-1:-1:-1;2048:37005:43;;;;;;;;;;:::o;:::-;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;:::i;:::-;;;;;;;;;:::i;:::-;;;;;;;;:::o;:::-;;;;;;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;:::o;:::-;;;;;;;;;:::o;13671:2339::-;;2048:37005;13935:48;13943:18;;;:::i;13935:48::-;14001:9;;2048:37005;;;;;14131:11;2048:37005;;;;;;14237:21;;;;:::i;:::-;14300:20;;;;:::i;:::-;14362;;;;:::i;:::-;14393:23;2048:37005;14426:23;2048:37005;14465:13;2048:37005;14480:14;;;;;;14460:665;15200:16;;;;;;;15196:808;15200:16;;;2048:37005;;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;;;:::i;:::-;;;;;;;;15262:69;;;2048:37005;15262:69;:::o;15196:808::-;15352:19;;;;;;;;15348:656;15352:19;;;15464:27;;;:::i;:::-;15516:26;;;;:::i;:::-;15567;;;;:::i;:::-;15613:13;2048:37005;15628:15;;;;;;15348:656;;;;;13671:2339::o;15645:3::-;15680:13;;2048:37005;15680:13;;;:::i;:::-;;15668:25;;;;:::i;:::-;;;;;;:::i;:::-;;15725:15;;;;:::i;:::-;2048:37005;15711:29;;;;:::i;:::-;2048:37005;15772:15;;;;:::i;:::-;2048:37005;15758:29;;;;:::i;:::-;2048:37005;;15613:13;;15348:656;15900:19;;;15348:656;13671:2339::o;14496:3::-;2048:37005;;;14519:15;2048:37005;;;;;;;;;;;;;;14515:600;;14496:3;2048:37005;;14465:13;;14515:600;14572:21;;;;;;:44;;;14515:600;14568:502;;;2048:37005;;;14666:9;2048:37005;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;:::i;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;14640:45;;;;:::i;:::-;;;;;;:::i;:::-;;14707:29;;;;;:::i;:::-;2048:37005;;;;14519:15;2048:37005;;;;;;;;;;;;;;;;14131:11;2048:37005;;;;;;;;;;;;;14838:13;14758:58;2048:37005;14758:58;;;;15087:13;14758:58;;:::i;:::-;2048:37005;14838:13;:::i;:::-;14568:502;15087:13;:::i;:::-;14515:600;;;;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;14131:11;2048:37005;;;;;;;14568:502;14895:14;;;;;:::i;:::-;14880:29;;;14876:194;;15087:13;2048:37005;14568:502;15087:13;:::i;14876:194::-;15010:14;;;;;;;2048:37005;15046:5;;;;;;;;14572:44;14597:19;;;;14572:44;;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::i;:::-;;;;;;;;:::i;:::-;;;;-1:-1:-1;2048:37005:43;;;;:::o;:::-;;;:::o;28852:406::-;-1:-1:-1;28852:406:43;;2048:37005;;29032:23;;;2048:37005;;;29032:23;;;;;;:::i;:::-;28999:57;;2435:42;28999:57;;;;:::i;:::-;2048:37005;;;;29032:23;2048:37005;;;29222:29;;2048:37005;;;;29032:23;29222:29;2048:37005;28852:406;:::o;2048:37005::-;;;;;;;29032:23;2048:37005;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;;;;;;38142:909;2048:37005;;;;38262:97;2048:37005;38262:97;;2779:66;38262:97;;;;;2048:37005;2580:5;2048:37005;;;;38262:97;2626:42;;38262:97;;;2048:37005;38262:97;;;;-1:-1:-1;;38262:97:43;;;38142:909;-1:-1:-1;38258:787:43;;38645:400;;;38758:26;2048:37005;38645:400;;:::i;:::-;2048:37005;;;;;;;;;;;;;:::i;:::-;38758:26;;;2048:37005;;;38836:42;;;;2048:37005;38836:42;;38262:97;38836:42;;2048:37005;38836:42;;;;;;;-1:-1:-1;38836:42:43;;;38258:787;2048:37005;;38892:80;-1:-1:-1;2048:37005:43;;;;38900:19;38892:80;:::i;:::-;2048:37005;;;;39019:14;;2048:37005;;;38986:48;:::o;38836:42::-;;;;;;;-1:-1:-1;38836:42:43;;;;;;;:::i;:::-;;;;;:::i;:::-;;;;;;;;;;;;2048:37005;;;-1:-1:-1;2048:37005:43;;;;;38258:787;2048:37005;;;38419:80;2048:37005;;-1:-1:-1;2048:37005:43;;;;;38427:19;38419:80;:::i;38262:97::-;;;;;;;;;;;;;;;:::i;:::-;;;;;2048:37005;;;;;;;;;;;;;:::o;:::-;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;;;8772:148;2048:37005;8860:13;2048:37005;;8852:21;;:61;;;;8845:68;8772:148;:::o;8852:61::-;2048:37005;;-1:-1:-1;2048:37005:43;8877:15;2048:37005;;;;-1:-1:-1;2048:37005:43;;;8877:36;;8772:148;:::o;2048:37005::-;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;3405:215:23:-;2048:37005:43;;;;3489:22:23;;;3485:91;;1280:65;2048:37005:43;;;;;;;;;;;3975:40:23;-1:-1:-1;3975:40:23;;3405:215::o;3485:91::-;2048:37005:43;;;3534:31:23;;;3509:1;3534:31;;;2048:37005:43;3534:31:23;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;:::o;196:563:39:-;274:3;2048:37005:43;;316:3:39;2048:37005:43;325:6:39;321:32;;196:563;2048:37005:43;;371:2:39;2048:37005:43;380:6:39;376:32;;196:563;2048:37005:43;;426:2:39;2048:37005:43;435:6:39;431:32;;196:563;2048:37005:43;;481:2:39;2048:37005:43;490:6:39;486:32;;196:563;2048:37005:43;;536:1:39;2048:37005:43;545:6:39;541:32;;196:563;2048:37005:43;;591:1:39;2048:37005:43;600:6:39;596:32;;196:563;2048:37005:43;;646:1:39;2048:37005:43;655:6:39;651:32;;196:563;2048:37005:43;;701:1:39;2048:37005:43;706:24:39;;747:5;;;:::i;:::-;196:563;:::o;706:24::-;2048:37005:43;;;;;;;;;718:12:39;:::o;651:32::-;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;675:5:39;651:32;;;596;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;620:5:39;596:32;;;541;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;565:5:39;541:32;;;486;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;510:5:39;486:32;;;431;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;455:5:39;431:32;;;376;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;;;400:5:39;376:32;;;321;335:8;;;316:3;345:5;321:32;;;35854:1341:43;;2048:37005;;;;;;;;:::i;:::-;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;;;-1:-1:-1;2048:37005:43;36013:16;2048:37005;;;-1:-1:-1;2048:37005:43;;36001:35;;2048:37005;;;36096:1;;2048:37005;;;;;;;;;;;;36075:34;;-1:-1:-1;36271:15:43;36266:617;;36288:5;;;2048:37005;;;-1:-1:-1;2048:37005:43;36571:13;2048:37005;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;36904:37;2048:37005;-1:-1:-1;2048:37005:43;;36904:37;;:::i;:::-;36955:22;;;;36951:152;;37172:15;;;;:::i;:::-;2048:37005;;;;;;:::i;:::-;;;37134:54;;2048:37005;35854:1341;:::o;36951:152::-;2048:37005;;36096:1;2048:37005;;;;;;;37070:21;;;:::i;36295:3::-;2048:37005;;;;-1:-1:-1;2048:37005:43;36435:11;2048:37005;;;-1:-1:-1;2048:37005:43;;36422:31;;;36418:121;;2048:37005;-1:-1:-1;2048:37005:43;36571:13;2048:37005;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;36565:37;2048:37005;-1:-1:-1;2048:37005:43;;36565:37;;:::i;:::-;36644:22;;;36686:38;2048:37005;;-1:-1:-1;2048:37005:43;;;;;-1:-1:-1;2048:37005:43;;-1:-1:-1;2048:37005:43;;;;-1:-1:-1;2048:37005:43;;36686:38;;:::i;:::-;2048:37005;;;;;;;;;;;;;36742:25;2048:37005;;36742:25;;:::i;:::-;36640:233;;2048:37005;;;;;;36271:15;;;;;2048:37005;;;;-1:-1:-1;2048:37005:43;;;-1:-1:-1;2048:37005:43;36640:233;2048:37005;;;;;;;;;;;;;36833:25;2048:37005;;36833:25;;:::i;:::-;36640:233;;;36418:121;2048:37005;;;;;;;;;;36473:25;2048:37005;;36473:25;;:::i;2048:37005::-;;;;;;;;;;;;;;;;;;;;;;;;277:15:42;;;;;;;;;;;:::o;1669:1834::-;;;;;1927:19;;;;:41;;;1669:1834;2048:37005:43;;;;;2051:15:42;;;2048:37005:43;;2145:11:42;;2048:37005:43;;;;;;1945:1:42;2326:20;;1945:1;;2467:45;2516:29;2467:45;:79;:45;;:::i;:::-;277:15;;;2516:29;:::i;:::-;2467:79;;:::i;:::-;1025:5;;;;2874:46;1025:5;2874:36;1025:5;2048:37005:43;1025:5:42;2874:36;:::i;:::-;:46;:::i;:::-;2048:37005:43;799:1:42;2048:37005:43;;;;;;;;799:1:42;2048:37005:43;;;;;;3043:3:42;1025:5;;2048:37005:43;;;;;;;689:1:42;2048:37005:43;;;;;;3152:32:42;;;;;3200:8;;;;;1945:1;3200:8;:::o;3148:349::-;3043:3;1025:5;;3277:31;;3043:3;;3331:31;;;;;:::i;3273:224::-;2048:37005:43;;;;;;235:1:42;2048:37005:43;;;;;3043:3:42;1025:5;;3426:60;:::o;2322:452::-;277:15;;;;;;;2665:30;277:15;1945:1;277:15;;;2665:30;:::i;:::-;2048:37005:43;;;;;;;;;;;;2717:45:42;;;:::i;:::-;1025:5;;;;;2874:46;1025:5;2874:36;1025:5;;2874:36;:::i;2048:37005:43:-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;1927:41:42;1950:13;;:18;1927:41;;2658:162:23;2048:37005:43;1280:65:23;2048:37005:43;;966:10:26;2717:23:23;2713:101;;2658:162::o;2713:101::-;2048:37005:43;;;2763:40:23;;;966:10:26;2763:40:23;;;2048:37005:43;2763:40:23;2048:37005:43;;;;;;;;:::o;:::-;;;;;;;;;;;;;;;;;;;;;:::o;7084:141:24:-;2048:37005:43;8837:64:24;2048:37005:43;;;;7150:18:24;7146:73;;7084:141::o;7146:73::-;7191:17;2048:37005:43;;7191:17:24;;;;4421:582:33;;4593:8;;-1:-1:-1;2048:37005:43;;5674:21:33;:17;;5799:158;;;;;;5670:354;5994:19;2048:37005:43;;5994:19:33;;;;4589:408;2048:37005:43;;4841:22:33;:49;;;4589:408;4837:119;;4969:17;;:::o;4837:119::-;2048:37005:43;;;;;4917:24:33;;;;2048:37005:43;4917:24:33;;;2048:37005:43;4917:24:33;4841:49;4867:18;;;:23;4841:49;;1613:793:39;1700:16;1687:30;;2048:37005:43;;1778:3:39;2048:37005:43;;;;;;;-1:-1:-1;2048:37005:43;1828:13:39;1855:6;1851:40;;1613:793;791:66;1904:11;;1900:55;;1613:793;888:66;1968:10;;1964:53;;1613:793;985:66;2030:10;;2026:53;;1613:793;1082:66;2092:10;;2088:53;;1613:793;1178:66;2154:9;;2150:51;;1613:793;1274:66;2214:9;;2210:51;;1613:793;1370:66;2274:9;;2270:51;;1613:793;1466:66;2334:9;2330:51;;1613:793;:::o;2330:51::-;2048:37005:43;;;;;;;;1613:793:39;:::o;2270:51::-;2048:37005:43;;;;;;;;;2304:6:39;2270:51;;2210;2048:37005:43;;;;;;;;;2210:51:39;;;2150;2048:37005:43;;;;;;;;;2150:51:39;;;2088:53;2048:37005:43;;;;;;;;;2088:53:39;;;2026;2048:37005:43;;;;;;;;;2026:53:39;;;1964;2048:37005:43;;;;;;;;;1964:53:39;;;1900:55;2048:37005:43;;;;;;;;;1900:55:39;;;1851:40;2048:37005:43;;-1:-1:-1;1851:40:39;;2048:37005:43;;;;;;;;;;;;;;;;;;;;;;;;;;;;10520:532:44;10615:431;;;;;;;;;;;;;;;;;;10520:532;:::o","linkReferences":{},"immutableReferences":{"40699":[{"start":10609,"length":32},{"start":10981,"length":32}]}},"methodIdentifiers":{"BURN_ACTOR()":"0a6a63f1","EXTRA_DATA_MAX_SIZE()":"029b4646","FIL_USD_PRICE_FEED_ID()":"19c75950","LEAF_SIZE()":"c0e15949","MAX_ENQUEUED_REMOVALS()":"9f8cb3bd","MAX_PIECE_SIZE()":"8a405abc","NO_CHALLENGE_SCHEDULED()":"462dd449","NO_PROVEN_EPOCH()":"f178b1be","PYTH()":"67e406d5","RANDOMNESS_PRECOMPILE()":"15b17570","SECONDS_IN_DAY()":"61a52a36","UPGRADE_INTERFACE_VERSION()":"ad3cb1cc","VERSION()":"ffa1ad74","addPieces(uint256,((bytes),uint256)[],bytes)":"ddea76cc","calculateProofFee(uint256,uint256)":"4903704a","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getActivePieceCount(uint256)":"5353bdfd","getActivePieces(uint256,uint256,uint256)":"39f51544","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getFILUSDPrice()":"4fa27920","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getRandomness(uint256)":"453f4f62","getScheduledRemovals(uint256)":"6fa44692","initialize(uint256)":"fe4b84df","migrate()":"8fd3ab80","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","owner()":"8da5cb5b","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","proxiableUUID()":"52d1902d","renounceOwnership()":"715018a6","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024","transferOwnership(address)":"f2fde38b","upgradeToAndCall(address,bytes)":"4f1ef286"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"target\",\"type\":\"address\"}],\"name\":\"AddressEmptyCode\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"ERC1967InvalidImplementation\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"ERC1967NonPayable\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"FailedCall\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"idx\",\"type\":\"uint256\"},{\"internalType\":\"string\",\"name\":\"msg\",\"type\":\"string\"}],\"name\":\"IndexedError\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"InvalidInitialization\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"NotInitializing\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"OwnableInvalidOwner\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"account\",\"type\":\"address\"}],\"name\":\"OwnableUnauthorizedAccount\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"UUPSUnauthorizedCallContext\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"slot\",\"type\":\"bytes32\"}],\"name\":\"UUPSUnsupportedProxiableUUID\",\"type\":\"error\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"version\",\"type\":\"uint64\"}],\"name\":\"Initialized\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"previousOwner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"OwnershipTransferred\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"reason\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"implementation\",\"type\":\"address\"}],\"name\":\"Upgraded\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"BURN_ACTOR\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"EXTRA_DATA_MAX_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"FIL_USD_PRICE_FEED_ID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"LEAF_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_ENQUEUED_REMOVALS\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"MAX_PIECE_SIZE\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_CHALLENGE_SCHEDULED\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"NO_PROVEN_EPOCH\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PYTH\",\"outputs\":[{\"internalType\":\"contract IPyth\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"RANDOMNESS_PRECOMPILE\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"SECONDS_IN_DAY\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"UPGRADE_INTERFACE_VERSION\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"VERSION\",\"outputs\":[{\"internalType\":\"string\",\"name\":\"\",\"type\":\"string\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"piece\",\"type\":\"tuple\"},{\"internalType\":\"uint256\",\"name\":\"rawSize\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceData[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"estimatedGasFee\",\"type\":\"uint256\"}],\"name\":\"calculateProofFee\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getActivePieceCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"activeCount\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"limit\",\"type\":\"uint256\"}],\"name\":\"getActivePieces\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieces\",\"type\":\"tuple[]\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"uint256[]\",\"name\":\"rawSizes\",\"type\":\"uint256[]\"},{\"internalType\":\"bool\",\"name\":\"hasMore\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getFILUSDPrice\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"},{\"internalType\":\"int32\",\"name\":\"\",\"type\":\"int32\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid\",\"name\":\"\",\"type\":\"tuple\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"epoch\",\"type\":\"uint256\"}],\"name\":\"getRandomness\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"_challengeFinality\",\"type\":\"uint256\"}],\"name\":\"initialize\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"migrate\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"owner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"proxiableUUID\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"renounceOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"transferOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"name\":\"upgradeToAndCall\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"}],\"devdoc\":{\"errors\":{\"AddressEmptyCode(address)\":[{\"details\":\"There's no code at `target` (it is not a contract).\"}],\"ERC1967InvalidImplementation(address)\":[{\"details\":\"The `implementation` of the proxy is invalid.\"}],\"ERC1967NonPayable()\":[{\"details\":\"An upgrade function sees `msg.value > 0` that may be lost.\"}],\"FailedCall()\":[{\"details\":\"A call to an address target failed. The target may have reverted.\"}],\"InvalidInitialization()\":[{\"details\":\"The contract is already initialized.\"}],\"NotInitializing()\":[{\"details\":\"The contract is not initializing.\"}],\"OwnableInvalidOwner(address)\":[{\"details\":\"The owner is not a valid owner account. (eg. `address(0)`)\"}],\"OwnableUnauthorizedAccount(address)\":[{\"details\":\"The caller account is not authorized to perform an operation.\"}],\"UUPSUnauthorizedCallContext()\":[{\"details\":\"The call is from an unauthorized context.\"}],\"UUPSUnsupportedProxiableUUID(bytes32)\":[{\"details\":\"The storage `slot` is unsupported as a UUID.\"}]},\"events\":{\"Initialized(uint64)\":{\"details\":\"Triggered when the contract has been initialized or reinitialized.\"},\"Upgraded(address)\":{\"details\":\"Emitted when the implementation is upgraded.\"}},\"kind\":\"dev\",\"methods\":{\"constructor\":{\"custom:oz-upgrades-unsafe-allow\":\"constructor\"},\"getActivePieceCount(uint256)\":{\"params\":{\"setId\":\"The data set ID\"},\"returns\":{\"activeCount\":\"The number of active pieces in the data set\"}},\"getActivePieces(uint256,uint256,uint256)\":{\"params\":{\"limit\":\"Maximum number of pieces to return\",\"offset\":\"Starting index for pagination (0-based)\",\"setId\":\"The data set ID\"},\"returns\":{\"hasMore\":\"True if there are more pieces beyond this page\",\"pieceIds\":\"Array of corresponding piece IDs\",\"pieces\":\"Array of active piece CIDs\",\"rawSizes\":\"Array of raw sizes for each piece (in bytes)\"}},\"owner()\":{\"details\":\"Returns the address of the current owner.\"},\"proxiableUUID()\":{\"details\":\"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier.\"},\"renounceOwnership()\":{\"details\":\"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner.\"},\"transferOwnership(address)\":{\"details\":\"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner.\"},\"upgradeToAndCall(address,bytes)\":{\"custom:oz-upgrades-unsafe-allow-reachable\":\"delegatecall\",\"details\":\"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event.\"}},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"getActivePieceCount(uint256)\":{\"notice\":\"Returns the count of active pieces (non-zero leaf count) for a data set\"},\"getActivePieces(uint256,uint256,uint256)\":{\"notice\":\"Returns active pieces (non-zero leaf count) for a data set with pagination\"}},\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/PDPVerifier.sol\":\"PDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol\":{\"keccak256\":\"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6\",\"dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol\":{\"keccak256\":\"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609\",\"dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol\":{\"keccak256\":\"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0\",\"dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB\"]},\"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol\":{\"keccak256\":\"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9\",\"dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV\"]},\"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol\":{\"keccak256\":\"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d\",\"dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7\"]},\"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol\":{\"keccak256\":\"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196\",\"dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA\"]},\"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol\":{\"keccak256\":\"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a\",\"dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw\"]},\"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol\":{\"keccak256\":\"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa\",\"dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM\"]},\"lib/openzeppelin-contracts/contracts/utils/Address.sol\":{\"keccak256\":\"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23\",\"dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb\"]},\"lib/openzeppelin-contracts/contracts/utils/Errors.sol\":{\"keccak256\":\"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf\",\"dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB\"]},\"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol\":{\"keccak256\":\"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b\",\"dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM\"]},\"lib/pyth-sdk-solidity/IPyth.sol\":{\"keccak256\":\"0x949c65c65fea0578c09a6fc068e09ed1165adede2c835984cefcb25d76de1de2\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://4d7cb071e08e81bb8b113a928f4c2d2b3cdf950ad64c6c7003ea3d874163ca77\",\"dweb:/ipfs/QmRbQchPxRTBMHi7WzLb8XnMGzPDQcWhu7i2u5naUsCRoZ\"]},\"lib/pyth-sdk-solidity/IPythEvents.sol\":{\"keccak256\":\"0x048a35526c2e77d107d43ba336f1dcf31f64cef25ba429ae1f7a0fbc11c23320\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://b75be4c3643b22305995aba71fc92146dbf51fa82d2f9728c515d7749b32dca3\",\"dweb:/ipfs/QmRby4XA9jJQGhxoJ16BTUDuU7BzLFfadbfTgBiQsDgNyZ\"]},\"lib/pyth-sdk-solidity/PythStructs.sol\":{\"keccak256\":\"0x95ff0a6d64517348ef604b8bcf246b561a9445d7e607b8f48491c617cfda9b65\",\"license\":\"Apache-2.0\",\"urls\":[\"bzz-raw://fb7f4ffe03be7379d3833c5946e38153de26aef4a4da0323a1ec603787de9eb7\",\"dweb:/ipfs/QmW4WkkLPGjDJrLrW4mYfxtFh8e9KAcPhrnNdxPQsfkS6t\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe9cae265ac70772a6e575e1aee25e046546d16fee65eac956e76aa2c2c4d5d29\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://d48ea8f414e95f53f19f137e5ee95bb327de01ef34be42f8b33a6bc3b88d2535\",\"dweb:/ipfs/QmcMUrKtTW6KpYiiLySBhKakWhUqzifspbzExzRxdoy2A3\"]},\"src/Cids.sol\":{\"keccak256\":\"0xcdfc21c273c5d123e83502248ccd652125ca06465b04a62b12141655090294d7\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://d6663a2a078f0e92fa2555047de4c1fca6eaf47c3448b7d7e53f49d73ed94df9\",\"dweb:/ipfs/QmR6HXrpYqHEKf82WWgVpifRUqygCnSYbSgJjUyt6NVQhg\"]},\"src/Fees.sol\":{\"keccak256\":\"0xbe931ac353310b1f507e30eb4785613169d87146043de6705ba29c9cce39fec2\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://2d09cdebf923161ca3a94a42c56e74a8038cef1b90e933ba48ad916ad89d9ecc\",\"dweb:/ipfs/QmZ6Cx99hAGvuQqkoqAdAuqeT6Fq4Z5Msw7HHGJ8U9it1D\"]},\"src/PDPVerifier.sol\":{\"keccak256\":\"0x6e3ceade2f31e32386e82fedc2faf64746469a7bfafd80b3216274c161fa4879\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://c9f7c353784ef400df67a0771ea5924d50e80e61d28aa65044d8a3d5678baae1\",\"dweb:/ipfs/QmW71sT7wNTLdBUcKknBvuYqsnGYVCmCU7RHydVyrdWSzs\"]},\"src/Proofs.sol\":{\"keccak256\":\"0x9c4a870d9b9d9ea55826fd8b2d2b377ce54f958652189f74a4361949c401069f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://207b6b1587e7ff73068d82b559566b05f52e228efd8fc4e32377d38b4ebd61c2\",\"dweb:/ipfs/QmUjpfXpqsewRVbNTXG4JvF2gVp7Wh17KkcNHbZSRrCdjM\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0x1d127464d67825e324a78214b2219261826a605e7accdbe8beff0fc624612495\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://cc3e1fe67a19aa899779b56a6dad6ed36db01ad8fcdc54667fff14737ebb3ac0\",\"dweb:/ipfs/QmU94UZX7YeaAoUSymuxBEkVg4tFz1S4g5ux7zfWWyVsLy\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x3e784eb8c8eb9fcb860c922e328c4e53d78d3b9bc6a570d0112ee520d2026b16\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://63d33e78ca97ca13878236a5ac6daee67f6fa3a413349a527d77aac3449a659f\",\"dweb:/ipfs/Qme1tSmw4txRwTBG3njFjqdjw2sJjUVnC7Vmwksej9wYKf\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"inputs":[{"internalType":"address","name":"target","type":"address"}],"type":"error","name":"AddressEmptyCode"},{"inputs":[{"internalType":"address","name":"implementation","type":"address"}],"type":"error","name":"ERC1967InvalidImplementation"},{"inputs":[],"type":"error","name":"ERC1967NonPayable"},{"inputs":[],"type":"error","name":"FailedCall"},{"inputs":[{"internalType":"uint256","name":"idx","type":"uint256"},{"internalType":"string","name":"msg","type":"string"}],"type":"error","name":"IndexedError"},{"inputs":[],"type":"error","name":"InvalidInitialization"},{"inputs":[],"type":"error","name":"NotInitializing"},{"inputs":[{"internalType":"address","name":"owner","type":"address"}],"type":"error","name":"OwnableInvalidOwner"},{"inputs":[{"internalType":"address","name":"account","type":"address"}],"type":"error","name":"OwnableUnauthorizedAccount"},{"inputs":[],"type":"error","name":"UUPSUnauthorizedCallContext"},{"inputs":[{"internalType":"bytes32","name":"slot","type":"bytes32"}],"type":"error","name":"UUPSUnsupportedProxiableUUID"},{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"implementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint64","name":"version","type":"uint64","indexed":false}],"type":"event","name":"Initialized","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"address","name":"previousOwner","type":"address","indexed":true},{"internalType":"address","name":"newOwner","type":"address","indexed":true}],"type":"event","name":"OwnershipTransferred","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"reason","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"address","name":"implementation","type":"address","indexed":true}],"type":"event","name":"Upgraded","anonymous":false},{"inputs":[],"stateMutability":"view","type":"function","name":"BURN_ACTOR","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"EXTRA_DATA_MAX_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"FIL_USD_PRICE_FEED_ID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"LEAF_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_ENQUEUED_REMOVALS","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"MAX_PIECE_SIZE","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_CHALLENGE_SCHEDULED","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"NO_PROVEN_EPOCH","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"PYTH","outputs":[{"internalType":"contract IPyth","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"RANDOMNESS_PRECOMPILE","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"SECONDS_IN_DAY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"UPGRADE_INTERFACE_VERSION","outputs":[{"internalType":"string","name":"","type":"string"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"VERSION","outputs":[{"internalType":"string","name":"","type":"string"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.PieceData[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"struct Cids.Cid","name":"piece","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"uint256","name":"rawSize","type":"uint256"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"estimatedGasFee","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"calculateProofFee","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getActivePieceCount","outputs":[{"internalType":"uint256","name":"activeCount","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"},{"internalType":"uint256","name":"limit","type":"uint256"}],"stateMutability":"view","type":"function","name":"getActivePieces","outputs":[{"internalType":"struct Cids.Cid[]","name":"pieces","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"uint256[]","name":"rawSizes","type":"uint256[]"},{"internalType":"bool","name":"hasMore","type":"bool"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"getFILUSDPrice","outputs":[{"internalType":"uint64","name":"","type":"uint64"},{"internalType":"int32","name":"","type":"int32"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"struct Cids.Cid","name":"","type":"tuple","components":[{"internalType":"bytes","name":"data","type":"bytes"}]}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"epoch","type":"uint256"}],"stateMutability":"view","type":"function","name":"getRandomness","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"_challengeFinality","type":"uint256"}],"stateMutability":"nonpayable","type":"function","name":"initialize"},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"migrate"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[],"stateMutability":"view","type":"function","name":"owner","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[],"stateMutability":"view","type":"function","name":"proxiableUUID","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}]},{"inputs":[],"stateMutability":"nonpayable","type":"function","name":"renounceOwnership"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"},{"inputs":[{"internalType":"address","name":"newOwner","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"transferOwnership"},{"inputs":[{"internalType":"address","name":"newImplementation","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"stateMutability":"payable","type":"function","name":"upgradeToAndCall"}],"devdoc":{"kind":"dev","methods":{"constructor":{"custom:oz-upgrades-unsafe-allow":"constructor"},"getActivePieceCount(uint256)":{"params":{"setId":"The data set ID"},"returns":{"activeCount":"The number of active pieces in the data set"}},"getActivePieces(uint256,uint256,uint256)":{"params":{"limit":"Maximum number of pieces to return","offset":"Starting index for pagination (0-based)","setId":"The data set ID"},"returns":{"hasMore":"True if there are more pieces beyond this page","pieceIds":"Array of corresponding piece IDs","pieces":"Array of active piece CIDs","rawSizes":"Array of raw sizes for each piece (in bytes)"}},"owner()":{"details":"Returns the address of the current owner."},"proxiableUUID()":{"details":"Implementation of the ERC-1822 {proxiableUUID} function. This returns the storage slot used by the implementation. It is used to validate the implementation's compatibility when performing an upgrade. IMPORTANT: A proxy pointing at a proxiable contract should not be considered proxiable itself, because this risks bricking a proxy that upgrades to it, by delegating to itself until out of gas. Thus it is critical that this function revert if invoked through a proxy. This is guaranteed by the `notDelegated` modifier."},"renounceOwnership()":{"details":"Leaves the contract without owner. It will not be possible to call `onlyOwner` functions. Can only be called by the current owner. NOTE: Renouncing ownership will leave the contract without an owner, thereby disabling any functionality that is only available to the owner."},"transferOwnership(address)":{"details":"Transfers ownership of the contract to a new account (`newOwner`). Can only be called by the current owner."},"upgradeToAndCall(address,bytes)":{"custom:oz-upgrades-unsafe-allow-reachable":"delegatecall","details":"Upgrade the implementation of the proxy to `newImplementation`, and subsequently execute the function call encoded in `data`. Calls {_authorizeUpgrade}. Emits an {Upgraded} event."}},"version":1},"userdoc":{"kind":"user","methods":{"getActivePieceCount(uint256)":{"notice":"Returns the count of active pieces (non-zero leaf count) for a data set"},"getActivePieces(uint256,uint256,uint256)":{"notice":"Returns active pieces (non-zero leaf count) for a data set with pagination"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/PDPVerifier.sol":"PDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts-upgradeable/contracts/access/OwnableUpgradeable.sol":{"keccak256":"0xc163fcf9bb10138631a9ba5564df1fa25db9adff73bd9ee868a8ae1858fe093a","urls":["bzz-raw://9706d43a0124053d9880f6e31a59f31bc0a6a3dc1acd66ce0a16e1111658c5f6","dweb:/ipfs/QmUFmfowzkRwGtDu36cXV9SPTBHJ3n7dG9xQiK5B28jTf2"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/Initializable.sol":{"keccak256":"0x631188737069917d2f909d29ce62c4d48611d326686ba6683e26b72a23bfac0b","urls":["bzz-raw://7a61054ae84cd6c4d04c0c4450ba1d6de41e27e0a2c4f1bcdf58f796b401c609","dweb:/ipfs/QmUvtdp7X1mRVyC3CsHrtPbgoqWaXHp3S1ZR24tpAQYJWM"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/proxy/utils/UUPSUpgradeable.sol":{"keccak256":"0x8816653b632f8f634b78885c35112232b44acbf6033ec9e5065d2dd94946b15a","urls":["bzz-raw://6c16be456b19a1dbaaff7e89b9f6f5c92a02544d5d5f89222a9f57b5a8cfc2f0","dweb:/ipfs/QmS4aeG6paPRwAM1puekhkyGR4mHuMUzFz3riVDv7fbvvB"],"license":"MIT"},"lib/openzeppelin-contracts-upgradeable/contracts/utils/ContextUpgradeable.sol":{"keccak256":"0xdbef5f0c787055227243a7318ef74c8a5a1108ca3a07f2b3a00ef67769e1e397","urls":["bzz-raw://08e39f23d5b4692f9a40803e53a8156b72b4c1f9902a88cd65ba964db103dab9","dweb:/ipfs/QmPKn6EYDgpga7KtpkA8wV2yJCYGMtc9K4LkJfhKX2RVSV"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/IERC1967.sol":{"keccak256":"0xb25a4f11fa80c702bf5cd85adec90e6f6f507f32f4a8e6f5dbc31e8c10029486","urls":["bzz-raw://6917f8a323e7811f041aecd4d9fd6e92455a6fba38a797ac6f6e208c7912b79d","dweb:/ipfs/QmShuYv55wYHGi4EFkDB8QfF7ZCHoKk2efyz3AWY1ExSq7"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/interfaces/draft-IERC1822.sol":{"keccak256":"0xc42facb5094f2f35f066a7155bda23545e39a3156faef3ddc00185544443ba7d","urls":["bzz-raw://d3b36282ab029b46bd082619a308a2ea11c309967b9425b7b7a6eb0b0c1c3196","dweb:/ipfs/QmP2YVfDB2FoREax3vJu7QhDnyYRMw52WPrCD4vdT2kuDA"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/ERC1967/ERC1967Utils.sol":{"keccak256":"0x02caa0e5f7bade9a0d8ad6058467d641cb67697cd4678c7b1c170686bafe9128","urls":["bzz-raw://33b42a434f5d5fdc5071be05238059b9d8938bdab510071a5c300a975abc405a","dweb:/ipfs/QmaThmoD3JMdHGhn4GUJbEGnKcojUG8PWMFoC7DFcQoeCw"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/proxy/beacon/IBeacon.sol":{"keccak256":"0xc59a78b07b44b2cf2e8ab4175fca91e8eca1eee2df7357b8d2a8833e5ea1f64c","urls":["bzz-raw://5aa4f07e65444784c29cd7bfcc2341b34381e4e5b5da9f0c5bd00d7f430e66fa","dweb:/ipfs/QmWRMh4Q9DpaU9GvsiXmDdoNYMyyece9if7hnfLz7uqzWM"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Address.sol":{"keccak256":"0x9d8da059267bac779a2dbbb9a26c2acf00ca83085e105d62d5d4ef96054a47f5","urls":["bzz-raw://c78e2aa4313323cecd1ef12a8d6265b96beee1a199923abf55d9a2a9e291ad23","dweb:/ipfs/QmUTs2KStXucZezzFo3EYeqYu47utu56qrF7jj1Gue65vb"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Errors.sol":{"keccak256":"0x6afa713bfd42cf0f7656efa91201007ac465e42049d7de1d50753a373648c123","urls":["bzz-raw://ba1d02f4847670a1b83dec9f7d37f0b0418d6043447b69f3a29a5f9efc547fcf","dweb:/ipfs/QmQ7iH2keLNUKgq2xSWcRmuBE5eZ3F5whYAkAGzCNNoEWB"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/StorageSlot.sol":{"keccak256":"0xcf74f855663ce2ae00ed8352666b7935f6cddea2932fdf2c3ecd30a9b1cd0e97","urls":["bzz-raw://9f660b1f351b757dfe01438e59888f31f33ded3afcf5cb5b0d9bf9aa6f320a8b","dweb:/ipfs/QmarDJ5hZEgBtCmmrVzEZWjub9769eD686jmzb2XpSU1cM"],"license":"MIT"},"lib/pyth-sdk-solidity/IPyth.sol":{"keccak256":"0x949c65c65fea0578c09a6fc068e09ed1165adede2c835984cefcb25d76de1de2","urls":["bzz-raw://4d7cb071e08e81bb8b113a928f4c2d2b3cdf950ad64c6c7003ea3d874163ca77","dweb:/ipfs/QmRbQchPxRTBMHi7WzLb8XnMGzPDQcWhu7i2u5naUsCRoZ"],"license":"Apache-2.0"},"lib/pyth-sdk-solidity/IPythEvents.sol":{"keccak256":"0x048a35526c2e77d107d43ba336f1dcf31f64cef25ba429ae1f7a0fbc11c23320","urls":["bzz-raw://b75be4c3643b22305995aba71fc92146dbf51fa82d2f9728c515d7749b32dca3","dweb:/ipfs/QmRby4XA9jJQGhxoJ16BTUDuU7BzLFfadbfTgBiQsDgNyZ"],"license":"Apache-2.0"},"lib/pyth-sdk-solidity/PythStructs.sol":{"keccak256":"0x95ff0a6d64517348ef604b8bcf246b561a9445d7e607b8f48491c617cfda9b65","urls":["bzz-raw://fb7f4ffe03be7379d3833c5946e38153de26aef4a4da0323a1ec603787de9eb7","dweb:/ipfs/QmW4WkkLPGjDJrLrW4mYfxtFh8e9KAcPhrnNdxPQsfkS6t"],"license":"Apache-2.0"},"src/BitOps.sol":{"keccak256":"0xe9cae265ac70772a6e575e1aee25e046546d16fee65eac956e76aa2c2c4d5d29","urls":["bzz-raw://d48ea8f414e95f53f19f137e5ee95bb327de01ef34be42f8b33a6bc3b88d2535","dweb:/ipfs/QmcMUrKtTW6KpYiiLySBhKakWhUqzifspbzExzRxdoy2A3"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0xcdfc21c273c5d123e83502248ccd652125ca06465b04a62b12141655090294d7","urls":["bzz-raw://d6663a2a078f0e92fa2555047de4c1fca6eaf47c3448b7d7e53f49d73ed94df9","dweb:/ipfs/QmR6HXrpYqHEKf82WWgVpifRUqygCnSYbSgJjUyt6NVQhg"],"license":"Apache-2.0 OR MIT"},"src/Fees.sol":{"keccak256":"0xbe931ac353310b1f507e30eb4785613169d87146043de6705ba29c9cce39fec2","urls":["bzz-raw://2d09cdebf923161ca3a94a42c56e74a8038cef1b90e933ba48ad916ad89d9ecc","dweb:/ipfs/QmZ6Cx99hAGvuQqkoqAdAuqeT6Fq4Z5Msw7HHGJ8U9it1D"],"license":"Apache-2.0 OR MIT"},"src/PDPVerifier.sol":{"keccak256":"0x6e3ceade2f31e32386e82fedc2faf64746469a7bfafd80b3216274c161fa4879","urls":["bzz-raw://c9f7c353784ef400df67a0771ea5924d50e80e61d28aa65044d8a3d5678baae1","dweb:/ipfs/QmW71sT7wNTLdBUcKknBvuYqsnGYVCmCU7RHydVyrdWSzs"],"license":"Apache-2.0 OR MIT"},"src/Proofs.sol":{"keccak256":"0x9c4a870d9b9d9ea55826fd8b2d2b377ce54f958652189f74a4361949c401069f","urls":["bzz-raw://207b6b1587e7ff73068d82b559566b05f52e228efd8fc4e32377d38b4ebd61c2","dweb:/ipfs/QmUjpfXpqsewRVbNTXG4JvF2gVp7Wh17KkcNHbZSRrCdjM"],"license":"MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0x1d127464d67825e324a78214b2219261826a605e7accdbe8beff0fc624612495","urls":["bzz-raw://cc3e1fe67a19aa899779b56a6dad6ed36db01ad8fcdc54667fff14737ebb3ac0","dweb:/ipfs/QmU94UZX7YeaAoUSymuxBEkVg4tFz1S4g5ux7zfWWyVsLy"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x3e784eb8c8eb9fcb860c922e328c4e53d78d3b9bc6a570d0112ee520d2026b16","urls":["bzz-raw://63d33e78ca97ca13878236a5ac6daee67f6fa3a413349a527d77aac3449a659f","dweb:/ipfs/Qme1tSmw4txRwTBG3njFjqdjw2sJjUVnC7Vmwksej9wYKf"],"license":"MIT"}},"version":1},"id":43} \ No newline at end of file +{"abi":[{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"newImplementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"errorData","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"addPieces(uint256,(bytes)[],bytes)":"306fc8be","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getScheduledRemovals(uint256)":"6fa44692","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"errorData\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"title\":\"IPDPVerifier\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"notice\":\"Main interface for the PDPVerifier contract\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/interfaces/IPDPVerifier.sol\":\"IPDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts/contracts/utils/Panic.sol\":{\"keccak256\":\"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a\",\"dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG\"]},\"lib/openzeppelin-contracts/contracts/utils/Strings.sol\":{\"keccak256\":\"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57\",\"dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn\"]},\"lib/openzeppelin-contracts/contracts/utils/math/Math.sol\":{\"keccak256\":\"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167\",\"dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol\":{\"keccak256\":\"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8\",\"dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol\":{\"keccak256\":\"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03\",\"dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04\",\"dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d\",\"dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0x9d2999603dc4662015b5d21d4efffbb174cffc67725eca71ff3e967377c201fb\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://292d27f6f6a15c940bc14c59bd3ea7055bf00ed0921fd60af7884d467b60f875\",\"dweb:/ipfs/QmaN9HnGHfhtuyJ5mZoWfrTMpwQ76jRDDhfBAJuuwLbLQ1\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f\",\"dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t\"]},\"src/interfaces/IPDPVerifier.sol\":{\"keccak256\":\"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3\",\"dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"newImplementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"errorData","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct Cids.Cid[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"bytes","name":"","type":"bytes"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/interfaces/IPDPVerifier.sol":"IPDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts/contracts/utils/Panic.sol":{"keccak256":"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a","urls":["bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a","dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Strings.sol":{"keccak256":"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2","urls":["bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57","dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/Math.sol":{"keccak256":"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f","urls":["bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167","dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol":{"keccak256":"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54","urls":["bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8","dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol":{"keccak256":"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3","urls":["bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03","dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ"],"license":"MIT"},"src/BitOps.sol":{"keccak256":"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc","urls":["bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04","dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9","urls":["bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d","dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j"],"license":"Apache-2.0 OR MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0x9d2999603dc4662015b5d21d4efffbb174cffc67725eca71ff3e967377c201fb","urls":["bzz-raw://292d27f6f6a15c940bc14c59bd3ea7055bf00ed0921fd60af7884d467b60f875","dweb:/ipfs/QmaN9HnGHfhtuyJ5mZoWfrTMpwQ76jRDDhfBAJuuwLbLQ1"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846","urls":["bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f","dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t"],"license":"MIT"},"src/interfaces/IPDPVerifier.sol":{"keccak256":"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437","urls":["bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3","dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc"],"license":"MIT"}},"version":1},"id":54} \ No newline at end of file diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index d718116f0..37c5c1fac 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -13,7 +13,7 @@ import ( const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" -const PDPTestNet = "Change Me" +const PDPTestNet = "CHANGEME" type PDPContracts struct { PDPVerifier common.Address diff --git a/pdp/contract/pdp_verifier.go b/pdp/contract/pdp_verifier.go index 9a9de2a37..170fa9b14 100644 --- a/pdp/contract/pdp_verifier.go +++ b/pdp/contract/pdp_verifier.go @@ -34,27 +34,21 @@ type CidsCid struct { Data []byte } -// PDPVerifierPieceData is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierPieceData struct { - Piece CidsCid - RawSize *big.Int -} - -// PDPVerifierPieceIdAndOffset is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierPieceIdAndOffset struct { +// IPDPTypesPieceIdAndOffset is an auto generated low-level Go binding around an user-defined struct. +type IPDPTypesPieceIdAndOffset struct { PieceId *big.Int Offset *big.Int } -// PDPVerifierProof is an auto generated low-level Go binding around an user-defined struct. -type PDPVerifierProof struct { +// IPDPTypesProof is an auto generated low-level Go binding around an user-defined struct. +type IPDPTypesProof struct { Leaf [32]byte Proof [][32]byte } // PDPVerifierMetaData contains all meta data concerning the PDPVerifier contract. var PDPVerifierMetaData = &bind.MetaData{ - ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.PieceData[]\",\"components\":[{\"name\":\"piece\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"rawSize\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structPDPVerifier.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"event\",\"name\":\"Debug\",\"inputs\":[{\"name\":\"message\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"value\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structPDPVerifier.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", + ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE_LOG2\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"activeCount\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"migrate\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"ContractUpgraded\",\"inputs\":[{\"name\":\"version\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":false,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PriceOracleFailure\",\"inputs\":[{\"name\":\"reason\",\"type\":\"bytes\",\"indexed\":false,\"internalType\":\"bytes\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", } // PDPVerifierABI is the input ABI used to generate the binding from. @@ -358,12 +352,12 @@ func (_PDPVerifier *PDPVerifierCallerSession) MAXENQUEUEDREMOVALS() (*big.Int, e return _PDPVerifier.Contract.MAXENQUEUEDREMOVALS(&_PDPVerifier.CallOpts) } -// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. +// MAXPIECESIZELOG2 is a free data retrieval call binding the contract method 0xf8eb8276. // -// Solidity: function MAX_PIECE_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) MAXPIECESIZE(opts *bind.CallOpts) (*big.Int, error) { +// Solidity: function MAX_PIECE_SIZE_LOG2() view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) MAXPIECESIZELOG2(opts *bind.CallOpts) (*big.Int, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "MAX_PIECE_SIZE") + err := _PDPVerifier.contract.Call(opts, &out, "MAX_PIECE_SIZE_LOG2") if err != nil { return *new(*big.Int), err @@ -375,18 +369,18 @@ func (_PDPVerifier *PDPVerifierCaller) MAXPIECESIZE(opts *bind.CallOpts) (*big.I } -// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. +// MAXPIECESIZELOG2 is a free data retrieval call binding the contract method 0xf8eb8276. // -// Solidity: function MAX_PIECE_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) MAXPIECESIZE() (*big.Int, error) { - return _PDPVerifier.Contract.MAXPIECESIZE(&_PDPVerifier.CallOpts) +// Solidity: function MAX_PIECE_SIZE_LOG2() view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) MAXPIECESIZELOG2() (*big.Int, error) { + return _PDPVerifier.Contract.MAXPIECESIZELOG2(&_PDPVerifier.CallOpts) } -// MAXPIECESIZE is a free data retrieval call binding the contract method 0x8a405abc. +// MAXPIECESIZELOG2 is a free data retrieval call binding the contract method 0xf8eb8276. // -// Solidity: function MAX_PIECE_SIZE() view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) MAXPIECESIZE() (*big.Int, error) { - return _PDPVerifier.Contract.MAXPIECESIZE(&_PDPVerifier.CallOpts) +// Solidity: function MAX_PIECE_SIZE_LOG2() view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) MAXPIECESIZELOG2() (*big.Int, error) { + return _PDPVerifier.Contract.MAXPIECESIZELOG2(&_PDPVerifier.CallOpts) } // NOCHALLENGESCHEDULED is a free data retrieval call binding the contract method 0x462dd449. @@ -575,35 +569,35 @@ func (_PDPVerifier *PDPVerifierCallerSession) UPGRADEINTERFACEVERSION() (string, return _PDPVerifier.Contract.UPGRADEINTERFACEVERSION(&_PDPVerifier.CallOpts) } -// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// VERSION is a free data retrieval call binding the contract method 0xffa1ad74. // -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) -func (_PDPVerifier *PDPVerifierCaller) CalculateProofFee(opts *bind.CallOpts, setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { +// Solidity: function VERSION() view returns(string) +func (_PDPVerifier *PDPVerifierCaller) VERSION(opts *bind.CallOpts) (string, error) { var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "calculateProofFee", setId, estimatedGasFee) + err := _PDPVerifier.contract.Call(opts, &out, "VERSION") if err != nil { - return *new(*big.Int), err + return *new(string), err } - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) + out0 := *abi.ConvertType(out[0], new(string)).(*string) return out0, err } -// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// VERSION is a free data retrieval call binding the contract method 0xffa1ad74. // -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) -func (_PDPVerifier *PDPVerifierSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.CallOpts, setId, estimatedGasFee) +// Solidity: function VERSION() view returns(string) +func (_PDPVerifier *PDPVerifierSession) VERSION() (string, error) { + return _PDPVerifier.Contract.VERSION(&_PDPVerifier.CallOpts) } -// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// VERSION is a free data retrieval call binding the contract method 0xffa1ad74. // -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) -func (_PDPVerifier *PDPVerifierCallerSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { - return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.CallOpts, setId, estimatedGasFee) +// Solidity: function VERSION() view returns(string) +func (_PDPVerifier *PDPVerifierCallerSession) VERSION() (string, error) { + return _PDPVerifier.Contract.VERSION(&_PDPVerifier.CallOpts) } // DataSetLive is a free data retrieval call binding the contract method 0xca759f27. @@ -640,15 +634,15 @@ func (_PDPVerifier *PDPVerifierCallerSession) DataSetLive(setId *big.Int) (bool, // FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. // // Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierCaller) FindPieceIds(opts *bind.CallOpts, setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { +func (_PDPVerifier *PDPVerifierCaller) FindPieceIds(opts *bind.CallOpts, setId *big.Int, leafIndexs []*big.Int) ([]IPDPTypesPieceIdAndOffset, error) { var out []interface{} err := _PDPVerifier.contract.Call(opts, &out, "findPieceIds", setId, leafIndexs) if err != nil { - return *new([]PDPVerifierPieceIdAndOffset), err + return *new([]IPDPTypesPieceIdAndOffset), err } - out0 := *abi.ConvertType(out[0], new([]PDPVerifierPieceIdAndOffset)).(*[]PDPVerifierPieceIdAndOffset) + out0 := *abi.ConvertType(out[0], new([]IPDPTypesPieceIdAndOffset)).(*[]IPDPTypesPieceIdAndOffset) return out0, err @@ -657,20 +651,20 @@ func (_PDPVerifier *PDPVerifierCaller) FindPieceIds(opts *bind.CallOpts, setId * // FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. // // Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { +func (_PDPVerifier *PDPVerifierSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]IPDPTypesPieceIdAndOffset, error) { return _PDPVerifier.Contract.FindPieceIds(&_PDPVerifier.CallOpts, setId, leafIndexs) } // FindPieceIds is a free data retrieval call binding the contract method 0x349c9179. // // Solidity: function findPieceIds(uint256 setId, uint256[] leafIndexs) view returns((uint256,uint256)[]) -func (_PDPVerifier *PDPVerifierCallerSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]PDPVerifierPieceIdAndOffset, error) { +func (_PDPVerifier *PDPVerifierCallerSession) FindPieceIds(setId *big.Int, leafIndexs []*big.Int) ([]IPDPTypesPieceIdAndOffset, error) { return _PDPVerifier.Contract.FindPieceIds(&_PDPVerifier.CallOpts, setId, leafIndexs) } // GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. // -// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256 activeCount) func (_PDPVerifier *PDPVerifierCaller) GetActivePieceCount(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} err := _PDPVerifier.contract.Call(opts, &out, "getActivePieceCount", setId) @@ -687,14 +681,14 @@ func (_PDPVerifier *PDPVerifierCaller) GetActivePieceCount(opts *bind.CallOpts, // GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. // -// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256 activeCount) func (_PDPVerifier *PDPVerifierSession) GetActivePieceCount(setId *big.Int) (*big.Int, error) { return _PDPVerifier.Contract.GetActivePieceCount(&_PDPVerifier.CallOpts, setId) } // GetActivePieceCount is a free data retrieval call binding the contract method 0x5353bdfd. // -// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256) +// Solidity: function getActivePieceCount(uint256 setId) view returns(uint256 activeCount) func (_PDPVerifier *PDPVerifierCallerSession) GetActivePieceCount(setId *big.Int) (*big.Int, error) { return _PDPVerifier.Contract.GetActivePieceCount(&_PDPVerifier.CallOpts, setId) } @@ -941,38 +935,6 @@ func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetStorageProvider(setId *b return _PDPVerifier.Contract.GetDataSetStorageProvider(&_PDPVerifier.CallOpts, setId) } -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierCaller) GetFILUSDPrice(opts *bind.CallOpts) (uint64, int32, error) { - var out []interface{} - err := _PDPVerifier.contract.Call(opts, &out, "getFILUSDPrice") - - if err != nil { - return *new(uint64), *new(int32), err - } - - out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) - out1 := *abi.ConvertType(out[1], new(int32)).(*int32) - - return out0, out1, err - -} - -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (uint64, int32, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) -} - -// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() view returns(uint64, int32) -func (_PDPVerifier *PDPVerifierCallerSession) GetFILUSDPrice() (uint64, int32, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) -} - // GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. // // Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) @@ -1314,27 +1276,48 @@ func (_PDPVerifier *PDPVerifierCallerSession) ProxiableUUID() ([32]byte, error) return _PDPVerifier.Contract.ProxiableUUID(&_PDPVerifier.CallOpts) } -// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. +// AddPieces is a paid mutator transaction binding the contract method 0x306fc8be. // -// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactor) AddPieces(opts *bind.TransactOpts, setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { +// Solidity: function addPieces(uint256 setId, (bytes)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactor) AddPieces(opts *bind.TransactOpts, setId *big.Int, pieceData []CidsCid, extraData []byte) (*types.Transaction, error) { return _PDPVerifier.contract.Transact(opts, "addPieces", setId, pieceData, extraData) } -// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. +// AddPieces is a paid mutator transaction binding the contract method 0x306fc8be. // -// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierSession) AddPieces(setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { +// Solidity: function addPieces(uint256 setId, (bytes)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierSession) AddPieces(setId *big.Int, pieceData []CidsCid, extraData []byte) (*types.Transaction, error) { return _PDPVerifier.Contract.AddPieces(&_PDPVerifier.TransactOpts, setId, pieceData, extraData) } -// AddPieces is a paid mutator transaction binding the contract method 0xddea76cc. +// AddPieces is a paid mutator transaction binding the contract method 0x306fc8be. // -// Solidity: function addPieces(uint256 setId, ((bytes),uint256)[] pieceData, bytes extraData) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactorSession) AddPieces(setId *big.Int, pieceData []PDPVerifierPieceData, extraData []byte) (*types.Transaction, error) { +// Solidity: function addPieces(uint256 setId, (bytes)[] pieceData, bytes extraData) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactorSession) AddPieces(setId *big.Int, pieceData []CidsCid, extraData []byte) (*types.Transaction, error) { return _PDPVerifier.Contract.AddPieces(&_PDPVerifier.TransactOpts, setId, pieceData, extraData) } +// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactor) CalculateProofFee(opts *bind.TransactOpts, setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "calculateProofFee", setId, estimatedGasFee) +} + +// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) +func (_PDPVerifier *PDPVerifierSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { + return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.TransactOpts, setId, estimatedGasFee) +} + +// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) +func (_PDPVerifier *PDPVerifierTransactorSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { + return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.TransactOpts, setId, estimatedGasFee) +} + // ClaimDataSetStorageProvider is a paid mutator transaction binding the contract method 0xdf0f3248. // // Solidity: function claimDataSetStorageProvider(uint256 setId, bytes extraData) returns() @@ -1398,6 +1381,27 @@ func (_PDPVerifier *PDPVerifierTransactorSession) DeleteDataSet(setId *big.Int, return _PDPVerifier.Contract.DeleteDataSet(&_PDPVerifier.TransactOpts, setId, extraData) } +// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() returns(uint64, int32) +func (_PDPVerifier *PDPVerifierTransactor) GetFILUSDPrice(opts *bind.TransactOpts) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "getFILUSDPrice") +} + +// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() returns(uint64, int32) +func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (*types.Transaction, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.TransactOpts) +} + +// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() returns(uint64, int32) +func (_PDPVerifier *PDPVerifierTransactorSession) GetFILUSDPrice() (*types.Transaction, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.TransactOpts) +} + // Initialize is a paid mutator transaction binding the contract method 0xfe4b84df. // // Solidity: function initialize(uint256 _challengeFinality) returns() @@ -1419,6 +1423,27 @@ func (_PDPVerifier *PDPVerifierTransactorSession) Initialize(_challengeFinality return _PDPVerifier.Contract.Initialize(&_PDPVerifier.TransactOpts, _challengeFinality) } +// Migrate is a paid mutator transaction binding the contract method 0x8fd3ab80. +// +// Solidity: function migrate() returns() +func (_PDPVerifier *PDPVerifierTransactor) Migrate(opts *bind.TransactOpts) (*types.Transaction, error) { + return _PDPVerifier.contract.Transact(opts, "migrate") +} + +// Migrate is a paid mutator transaction binding the contract method 0x8fd3ab80. +// +// Solidity: function migrate() returns() +func (_PDPVerifier *PDPVerifierSession) Migrate() (*types.Transaction, error) { + return _PDPVerifier.Contract.Migrate(&_PDPVerifier.TransactOpts) +} + +// Migrate is a paid mutator transaction binding the contract method 0x8fd3ab80. +// +// Solidity: function migrate() returns() +func (_PDPVerifier *PDPVerifierTransactorSession) Migrate() (*types.Transaction, error) { + return _PDPVerifier.Contract.Migrate(&_PDPVerifier.TransactOpts) +} + // NextProvingPeriod is a paid mutator transaction binding the contract method 0x45c0b92d. // // Solidity: function nextProvingPeriod(uint256 setId, uint256 challengeEpoch, bytes extraData) returns() @@ -1464,21 +1489,21 @@ func (_PDPVerifier *PDPVerifierTransactorSession) ProposeDataSetStorageProvider( // ProvePossession is a paid mutator transaction binding the contract method 0xf58f952b. // // Solidity: function provePossession(uint256 setId, (bytes32,bytes32[])[] proofs) payable returns() -func (_PDPVerifier *PDPVerifierTransactor) ProvePossession(opts *bind.TransactOpts, setId *big.Int, proofs []PDPVerifierProof) (*types.Transaction, error) { +func (_PDPVerifier *PDPVerifierTransactor) ProvePossession(opts *bind.TransactOpts, setId *big.Int, proofs []IPDPTypesProof) (*types.Transaction, error) { return _PDPVerifier.contract.Transact(opts, "provePossession", setId, proofs) } // ProvePossession is a paid mutator transaction binding the contract method 0xf58f952b. // // Solidity: function provePossession(uint256 setId, (bytes32,bytes32[])[] proofs) payable returns() -func (_PDPVerifier *PDPVerifierSession) ProvePossession(setId *big.Int, proofs []PDPVerifierProof) (*types.Transaction, error) { +func (_PDPVerifier *PDPVerifierSession) ProvePossession(setId *big.Int, proofs []IPDPTypesProof) (*types.Transaction, error) { return _PDPVerifier.Contract.ProvePossession(&_PDPVerifier.TransactOpts, setId, proofs) } // ProvePossession is a paid mutator transaction binding the contract method 0xf58f952b. // // Solidity: function provePossession(uint256 setId, (bytes32,bytes32[])[] proofs) payable returns() -func (_PDPVerifier *PDPVerifierTransactorSession) ProvePossession(setId *big.Int, proofs []PDPVerifierProof) (*types.Transaction, error) { +func (_PDPVerifier *PDPVerifierTransactorSession) ProvePossession(setId *big.Int, proofs []IPDPTypesProof) (*types.Transaction, error) { return _PDPVerifier.Contract.ProvePossession(&_PDPVerifier.TransactOpts, setId, proofs) } @@ -1566,9 +1591,9 @@ func (_PDPVerifier *PDPVerifierTransactorSession) UpgradeToAndCall(newImplementa return _PDPVerifier.Contract.UpgradeToAndCall(&_PDPVerifier.TransactOpts, newImplementation, data) } -// PDPVerifierDataSetCreatedIterator is returned from FilterDataSetCreated and is used to iterate over the raw logs and unpacked data for DataSetCreated events raised by the PDPVerifier contract. -type PDPVerifierDataSetCreatedIterator struct { - Event *PDPVerifierDataSetCreated // Event containing the contract specifics and raw log +// PDPVerifierContractUpgradedIterator is returned from FilterContractUpgraded and is used to iterate over the raw logs and unpacked data for ContractUpgraded events raised by the PDPVerifier contract. +type PDPVerifierContractUpgradedIterator struct { + Event *PDPVerifierContractUpgraded // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1582,7 +1607,7 @@ type PDPVerifierDataSetCreatedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierDataSetCreatedIterator) Next() bool { +func (it *PDPVerifierContractUpgradedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1591,7 +1616,7 @@ func (it *PDPVerifierDataSetCreatedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetCreated) + it.Event = new(PDPVerifierContractUpgraded) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1606,7 +1631,7 @@ func (it *PDPVerifierDataSetCreatedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetCreated) + it.Event = new(PDPVerifierContractUpgraded) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1622,60 +1647,42 @@ func (it *PDPVerifierDataSetCreatedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierDataSetCreatedIterator) Error() error { +func (it *PDPVerifierContractUpgradedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierDataSetCreatedIterator) Close() error { +func (it *PDPVerifierContractUpgradedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierDataSetCreated represents a DataSetCreated event raised by the PDPVerifier contract. -type PDPVerifierDataSetCreated struct { - SetId *big.Int - StorageProvider common.Address - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierContractUpgraded represents a ContractUpgraded event raised by the PDPVerifier contract. +type PDPVerifierContractUpgraded struct { + Version string + Implementation common.Address + Raw types.Log // Blockchain specific contextual infos } -// FilterDataSetCreated is a free log retrieval operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. +// FilterContractUpgraded is a free log retrieval operation binding the contract event 0x2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c3. // -// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) -func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetCreated(opts *bind.FilterOpts, setId []*big.Int, storageProvider []common.Address) (*PDPVerifierDataSetCreatedIterator, error) { +// Solidity: event ContractUpgraded(string version, address implementation) +func (_PDPVerifier *PDPVerifierFilterer) FilterContractUpgraded(opts *bind.FilterOpts) (*PDPVerifierContractUpgradedIterator, error) { - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) - } - var storageProviderRule []interface{} - for _, storageProviderItem := range storageProvider { - storageProviderRule = append(storageProviderRule, storageProviderItem) - } - - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "ContractUpgraded") if err != nil { return nil, err } - return &PDPVerifierDataSetCreatedIterator{contract: _PDPVerifier.contract, event: "DataSetCreated", logs: logs, sub: sub}, nil + return &PDPVerifierContractUpgradedIterator{contract: _PDPVerifier.contract, event: "ContractUpgraded", logs: logs, sub: sub}, nil } -// WatchDataSetCreated is a free log subscription operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. +// WatchContractUpgraded is a free log subscription operation binding the contract event 0x2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c3. // -// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) -func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetCreated(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetCreated, setId []*big.Int, storageProvider []common.Address) (event.Subscription, error) { - - var setIdRule []interface{} - for _, setIdItem := range setId { - setIdRule = append(setIdRule, setIdItem) - } - var storageProviderRule []interface{} - for _, storageProviderItem := range storageProvider { - storageProviderRule = append(storageProviderRule, storageProviderItem) - } +// Solidity: event ContractUpgraded(string version, address implementation) +func (_PDPVerifier *PDPVerifierFilterer) WatchContractUpgraded(opts *bind.WatchOpts, sink chan<- *PDPVerifierContractUpgraded) (event.Subscription, error) { - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "ContractUpgraded") if err != nil { return nil, err } @@ -1685,8 +1692,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetCreated(opts *bind.WatchOpt select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierDataSetCreated) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { + event := new(PDPVerifierContractUpgraded) + if err := _PDPVerifier.contract.UnpackLog(event, "ContractUpgraded", log); err != nil { return err } event.Raw = log @@ -1707,21 +1714,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetCreated(opts *bind.WatchOpt }), nil } -// ParseDataSetCreated is a log parse operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. +// ParseContractUpgraded is a log parse operation binding the contract event 0x2b51ff7c4cc8e6fe1c72e9d9685b7d2a88a5d82ad3a644afbdceb0272c89c1c3. // -// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) -func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetCreated(log types.Log) (*PDPVerifierDataSetCreated, error) { - event := new(PDPVerifierDataSetCreated) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { +// Solidity: event ContractUpgraded(string version, address implementation) +func (_PDPVerifier *PDPVerifierFilterer) ParseContractUpgraded(log types.Log) (*PDPVerifierContractUpgraded, error) { + event := new(PDPVerifierContractUpgraded) + if err := _PDPVerifier.contract.UnpackLog(event, "ContractUpgraded", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierDataSetDeletedIterator is returned from FilterDataSetDeleted and is used to iterate over the raw logs and unpacked data for DataSetDeleted events raised by the PDPVerifier contract. -type PDPVerifierDataSetDeletedIterator struct { - Event *PDPVerifierDataSetDeleted // Event containing the contract specifics and raw log +// PDPVerifierDataSetCreatedIterator is returned from FilterDataSetCreated and is used to iterate over the raw logs and unpacked data for DataSetCreated events raised by the PDPVerifier contract. +type PDPVerifierDataSetCreatedIterator struct { + Event *PDPVerifierDataSetCreated // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1735,7 +1742,7 @@ type PDPVerifierDataSetDeletedIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierDataSetDeletedIterator) Next() bool { +func (it *PDPVerifierDataSetCreatedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1744,7 +1751,7 @@ func (it *PDPVerifierDataSetDeletedIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetDeleted) + it.Event = new(PDPVerifierDataSetCreated) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1759,7 +1766,7 @@ func (it *PDPVerifierDataSetDeletedIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetDeleted) + it.Event = new(PDPVerifierDataSetCreated) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1775,52 +1782,60 @@ func (it *PDPVerifierDataSetDeletedIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierDataSetDeletedIterator) Error() error { +func (it *PDPVerifierDataSetCreatedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierDataSetDeletedIterator) Close() error { +func (it *PDPVerifierDataSetCreatedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierDataSetDeleted represents a DataSetDeleted event raised by the PDPVerifier contract. -type PDPVerifierDataSetDeleted struct { - SetId *big.Int - DeletedLeafCount *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetCreated represents a DataSetCreated event raised by the PDPVerifier contract. +type PDPVerifierDataSetCreated struct { + SetId *big.Int + StorageProvider common.Address + Raw types.Log // Blockchain specific contextual infos } -// FilterDataSetDeleted is a free log retrieval operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. +// FilterDataSetCreated is a free log retrieval operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetDeleted(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetDeletedIterator, error) { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetCreated(opts *bind.FilterOpts, setId []*big.Int, storageProvider []common.Address) (*PDPVerifierDataSetCreatedIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } + var storageProviderRule []interface{} + for _, storageProviderItem := range storageProvider { + storageProviderRule = append(storageProviderRule, storageProviderItem) + } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetDeleted", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) if err != nil { return nil, err } - return &PDPVerifierDataSetDeletedIterator{contract: _PDPVerifier.contract, event: "DataSetDeleted", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetCreatedIterator{contract: _PDPVerifier.contract, event: "DataSetCreated", logs: logs, sub: sub}, nil } -// WatchDataSetDeleted is a free log subscription operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. +// WatchDataSetCreated is a free log subscription operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetDeleted(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetDeleted, setId []*big.Int) (event.Subscription, error) { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetCreated(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetCreated, setId []*big.Int, storageProvider []common.Address) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } + var storageProviderRule []interface{} + for _, storageProviderItem := range storageProvider { + storageProviderRule = append(storageProviderRule, storageProviderItem) + } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetDeleted", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetCreated", setIdRule, storageProviderRule) if err != nil { return nil, err } @@ -1830,8 +1845,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetDeleted(opts *bind.WatchOpt select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierDataSetDeleted) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { + event := new(PDPVerifierDataSetCreated) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { return err } event.Raw = log @@ -1852,21 +1867,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetDeleted(opts *bind.WatchOpt }), nil } -// ParseDataSetDeleted is a log parse operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. +// ParseDataSetCreated is a log parse operation binding the contract event 0x11369440e1b7135015c16acb9bc14b55b0f4b23b02010c363d34aec2e5b96281. // -// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) -func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetDeleted(log types.Log) (*PDPVerifierDataSetDeleted, error) { - event := new(PDPVerifierDataSetDeleted) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { +// Solidity: event DataSetCreated(uint256 indexed setId, address indexed storageProvider) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetCreated(log types.Log) (*PDPVerifierDataSetCreated, error) { + event := new(PDPVerifierDataSetCreated) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetCreated", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierDataSetEmptyIterator is returned from FilterDataSetEmpty and is used to iterate over the raw logs and unpacked data for DataSetEmpty events raised by the PDPVerifier contract. -type PDPVerifierDataSetEmptyIterator struct { - Event *PDPVerifierDataSetEmpty // Event containing the contract specifics and raw log +// PDPVerifierDataSetDeletedIterator is returned from FilterDataSetDeleted and is used to iterate over the raw logs and unpacked data for DataSetDeleted events raised by the PDPVerifier contract. +type PDPVerifierDataSetDeletedIterator struct { + Event *PDPVerifierDataSetDeleted // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -1880,7 +1895,7 @@ type PDPVerifierDataSetEmptyIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierDataSetEmptyIterator) Next() bool { +func (it *PDPVerifierDataSetDeletedIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -1889,7 +1904,7 @@ func (it *PDPVerifierDataSetEmptyIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetEmpty) + it.Event = new(PDPVerifierDataSetDeleted) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1904,7 +1919,7 @@ func (it *PDPVerifierDataSetEmptyIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierDataSetEmpty) + it.Event = new(PDPVerifierDataSetDeleted) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -1920,51 +1935,52 @@ func (it *PDPVerifierDataSetEmptyIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierDataSetEmptyIterator) Error() error { +func (it *PDPVerifierDataSetDeletedIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierDataSetEmptyIterator) Close() error { +func (it *PDPVerifierDataSetDeletedIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierDataSetEmpty represents a DataSetEmpty event raised by the PDPVerifier contract. -type PDPVerifierDataSetEmpty struct { - SetId *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetDeleted represents a DataSetDeleted event raised by the PDPVerifier contract. +type PDPVerifierDataSetDeleted struct { + SetId *big.Int + DeletedLeafCount *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterDataSetEmpty is a free log retrieval operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. +// FilterDataSetDeleted is a free log retrieval operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event DataSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetEmpty(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetEmptyIterator, error) { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetDeleted(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetDeletedIterator, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetEmpty", setIdRule) + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetDeleted", setIdRule) if err != nil { return nil, err } - return &PDPVerifierDataSetEmptyIterator{contract: _PDPVerifier.contract, event: "DataSetEmpty", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetDeletedIterator{contract: _PDPVerifier.contract, event: "DataSetDeleted", logs: logs, sub: sub}, nil } -// WatchDataSetEmpty is a free log subscription operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. +// WatchDataSetDeleted is a free log subscription operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event DataSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetEmpty(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetEmpty, setId []*big.Int) (event.Subscription, error) { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetDeleted(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetDeleted, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} for _, setIdItem := range setId { setIdRule = append(setIdRule, setIdItem) } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetEmpty", setIdRule) + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetDeleted", setIdRule) if err != nil { return nil, err } @@ -1974,8 +1990,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetEmpty(opts *bind.WatchOpts, select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierDataSetEmpty) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { + event := new(PDPVerifierDataSetDeleted) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { return err } event.Raw = log @@ -1996,21 +2012,21 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetEmpty(opts *bind.WatchOpts, }), nil } -// ParseDataSetEmpty is a log parse operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. +// ParseDataSetDeleted is a log parse operation binding the contract event 0x14eeeef7679fcb051c6572811f61c07bedccd0f1cfc1f9b79b23e47c5c52aeb7. // -// Solidity: event DataSetEmpty(uint256 indexed setId) -func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetEmpty(log types.Log) (*PDPVerifierDataSetEmpty, error) { - event := new(PDPVerifierDataSetEmpty) - if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { +// Solidity: event DataSetDeleted(uint256 indexed setId, uint256 deletedLeafCount) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetDeleted(log types.Log) (*PDPVerifierDataSetDeleted, error) { + event := new(PDPVerifierDataSetDeleted) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetDeleted", log); err != nil { return nil, err } event.Raw = log return event, nil } -// PDPVerifierDebugIterator is returned from FilterDebug and is used to iterate over the raw logs and unpacked data for Debug events raised by the PDPVerifier contract. -type PDPVerifierDebugIterator struct { - Event *PDPVerifierDebug // Event containing the contract specifics and raw log +// PDPVerifierDataSetEmptyIterator is returned from FilterDataSetEmpty and is used to iterate over the raw logs and unpacked data for DataSetEmpty events raised by the PDPVerifier contract. +type PDPVerifierDataSetEmptyIterator struct { + Event *PDPVerifierDataSetEmpty // Event containing the contract specifics and raw log contract *bind.BoundContract // Generic contract to use for unpacking event data event string // Event name to use for unpacking event data @@ -2024,7 +2040,7 @@ type PDPVerifierDebugIterator struct { // Next advances the iterator to the subsequent event, returning whether there // are any more events found. In case of a retrieval or parsing error, false is // returned and Error() can be queried for the exact failure. -func (it *PDPVerifierDebugIterator) Next() bool { +func (it *PDPVerifierDataSetEmptyIterator) Next() bool { // If the iterator failed, stop iterating if it.fail != nil { return false @@ -2033,7 +2049,7 @@ func (it *PDPVerifierDebugIterator) Next() bool { if it.done { select { case log := <-it.logs: - it.Event = new(PDPVerifierDebug) + it.Event = new(PDPVerifierDataSetEmpty) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2048,7 +2064,7 @@ func (it *PDPVerifierDebugIterator) Next() bool { // Iterator still in progress, wait for either a data or an error event select { case log := <-it.logs: - it.Event = new(PDPVerifierDebug) + it.Event = new(PDPVerifierDataSetEmpty) if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { it.fail = err return false @@ -2064,42 +2080,51 @@ func (it *PDPVerifierDebugIterator) Next() bool { } // Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierDebugIterator) Error() error { +func (it *PDPVerifierDataSetEmptyIterator) Error() error { return it.fail } // Close terminates the iteration process, releasing any pending underlying // resources. -func (it *PDPVerifierDebugIterator) Close() error { +func (it *PDPVerifierDataSetEmptyIterator) Close() error { it.sub.Unsubscribe() return nil } -// PDPVerifierDebug represents a Debug event raised by the PDPVerifier contract. -type PDPVerifierDebug struct { - Message string - Value *big.Int - Raw types.Log // Blockchain specific contextual infos +// PDPVerifierDataSetEmpty represents a DataSetEmpty event raised by the PDPVerifier contract. +type PDPVerifierDataSetEmpty struct { + SetId *big.Int + Raw types.Log // Blockchain specific contextual infos } -// FilterDebug is a free log retrieval operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// FilterDataSetEmpty is a free log retrieval operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) FilterDebug(opts *bind.FilterOpts) (*PDPVerifierDebugIterator, error) { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) FilterDataSetEmpty(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierDataSetEmptyIterator, error) { - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "Debug") + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } + + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "DataSetEmpty", setIdRule) if err != nil { return nil, err } - return &PDPVerifierDebugIterator{contract: _PDPVerifier.contract, event: "Debug", logs: logs, sub: sub}, nil + return &PDPVerifierDataSetEmptyIterator{contract: _PDPVerifier.contract, event: "DataSetEmpty", logs: logs, sub: sub}, nil } -// WatchDebug is a free log subscription operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// WatchDataSetEmpty is a free log subscription operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink chan<- *PDPVerifierDebug) (event.Subscription, error) { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) WatchDataSetEmpty(opts *bind.WatchOpts, sink chan<- *PDPVerifierDataSetEmpty, setId []*big.Int) (event.Subscription, error) { + + var setIdRule []interface{} + for _, setIdItem := range setId { + setIdRule = append(setIdRule, setIdItem) + } - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "Debug") + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "DataSetEmpty", setIdRule) if err != nil { return nil, err } @@ -2109,8 +2134,8 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink c select { case log := <-logs: // New log arrived, parse the event and forward to the user - event := new(PDPVerifierDebug) - if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { + event := new(PDPVerifierDataSetEmpty) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { return err } event.Raw = log @@ -2131,12 +2156,12 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchDebug(opts *bind.WatchOpts, sink c }), nil } -// ParseDebug is a log parse operation binding the contract event 0x3c5ad147104e56be34a9176a6692f7df8d2f4b29a5af06bc6b98970d329d6577. +// ParseDataSetEmpty is a log parse operation binding the contract event 0x02a8400fc343f45098cb00c3a6ea694174771939a5503f663e0ff6f4eb7c2842. // -// Solidity: event Debug(string message, uint256 value) -func (_PDPVerifier *PDPVerifierFilterer) ParseDebug(log types.Log) (*PDPVerifierDebug, error) { - event := new(PDPVerifierDebug) - if err := _PDPVerifier.contract.UnpackLog(event, "Debug", log); err != nil { +// Solidity: event DataSetEmpty(uint256 indexed setId) +func (_PDPVerifier *PDPVerifierFilterer) ParseDataSetEmpty(log types.Log) (*PDPVerifierDataSetEmpty, error) { + event := new(PDPVerifierDataSetEmpty) + if err := _PDPVerifier.contract.UnpackLog(event, "DataSetEmpty", log); err != nil { return nil, err } event.Raw = log @@ -2936,7 +2961,7 @@ func (it *PDPVerifierPossessionProvenIterator) Close() error { // PDPVerifierPossessionProven represents a PossessionProven event raised by the PDPVerifier contract. type PDPVerifierPossessionProven struct { SetId *big.Int - Challenges []PDPVerifierPieceIdAndOffset + Challenges []IPDPTypesPieceIdAndOffset Raw types.Log // Blockchain specific contextual infos } @@ -3011,6 +3036,140 @@ func (_PDPVerifier *PDPVerifierFilterer) ParsePossessionProven(log types.Log) (* return event, nil } +// PDPVerifierPriceOracleFailureIterator is returned from FilterPriceOracleFailure and is used to iterate over the raw logs and unpacked data for PriceOracleFailure events raised by the PDPVerifier contract. +type PDPVerifierPriceOracleFailureIterator struct { + Event *PDPVerifierPriceOracleFailure // Event containing the contract specifics and raw log + + contract *bind.BoundContract // Generic contract to use for unpacking event data + event string // Event name to use for unpacking event data + + logs chan types.Log // Log channel receiving the found contract events + sub ethereum.Subscription // Subscription for errors, completion and termination + done bool // Whether the subscription completed delivering logs + fail error // Occurred error to stop iteration +} + +// Next advances the iterator to the subsequent event, returning whether there +// are any more events found. In case of a retrieval or parsing error, false is +// returned and Error() can be queried for the exact failure. +func (it *PDPVerifierPriceOracleFailureIterator) Next() bool { + // If the iterator failed, stop iterating + if it.fail != nil { + return false + } + // If the iterator completed, deliver directly whatever's available + if it.done { + select { + case log := <-it.logs: + it.Event = new(PDPVerifierPriceOracleFailure) + if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { + it.fail = err + return false + } + it.Event.Raw = log + return true + + default: + return false + } + } + // Iterator still in progress, wait for either a data or an error event + select { + case log := <-it.logs: + it.Event = new(PDPVerifierPriceOracleFailure) + if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { + it.fail = err + return false + } + it.Event.Raw = log + return true + + case err := <-it.sub.Err(): + it.done = true + it.fail = err + return it.Next() + } +} + +// Error returns any retrieval or parsing error occurred during filtering. +func (it *PDPVerifierPriceOracleFailureIterator) Error() error { + return it.fail +} + +// Close terminates the iteration process, releasing any pending underlying +// resources. +func (it *PDPVerifierPriceOracleFailureIterator) Close() error { + it.sub.Unsubscribe() + return nil +} + +// PDPVerifierPriceOracleFailure represents a PriceOracleFailure event raised by the PDPVerifier contract. +type PDPVerifierPriceOracleFailure struct { + Reason []byte + Raw types.Log // Blockchain specific contextual infos +} + +// FilterPriceOracleFailure is a free log retrieval operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. +// +// Solidity: event PriceOracleFailure(bytes reason) +func (_PDPVerifier *PDPVerifierFilterer) FilterPriceOracleFailure(opts *bind.FilterOpts) (*PDPVerifierPriceOracleFailureIterator, error) { + + logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PriceOracleFailure") + if err != nil { + return nil, err + } + return &PDPVerifierPriceOracleFailureIterator{contract: _PDPVerifier.contract, event: "PriceOracleFailure", logs: logs, sub: sub}, nil +} + +// WatchPriceOracleFailure is a free log subscription operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. +// +// Solidity: event PriceOracleFailure(bytes reason) +func (_PDPVerifier *PDPVerifierFilterer) WatchPriceOracleFailure(opts *bind.WatchOpts, sink chan<- *PDPVerifierPriceOracleFailure) (event.Subscription, error) { + + logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PriceOracleFailure") + if err != nil { + return nil, err + } + return event.NewSubscription(func(quit <-chan struct{}) error { + defer sub.Unsubscribe() + for { + select { + case log := <-logs: + // New log arrived, parse the event and forward to the user + event := new(PDPVerifierPriceOracleFailure) + if err := _PDPVerifier.contract.UnpackLog(event, "PriceOracleFailure", log); err != nil { + return err + } + event.Raw = log + + select { + case sink <- event: + case err := <-sub.Err(): + return err + case <-quit: + return nil + } + case err := <-sub.Err(): + return err + case <-quit: + return nil + } + } + }), nil +} + +// ParsePriceOracleFailure is a log parse operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. +// +// Solidity: event PriceOracleFailure(bytes reason) +func (_PDPVerifier *PDPVerifierFilterer) ParsePriceOracleFailure(log types.Log) (*PDPVerifierPriceOracleFailure, error) { + event := new(PDPVerifierPriceOracleFailure) + if err := _PDPVerifier.contract.UnpackLog(event, "PriceOracleFailure", log); err != nil { + return nil, err + } + event.Raw = log + return event, nil +} + // PDPVerifierProofFeePaidIterator is returned from FilterProofFeePaid and is used to iterate over the raw logs and unpacked data for ProofFeePaid events raised by the PDPVerifier contract. type PDPVerifierProofFeePaidIterator struct { Event *PDPVerifierProofFeePaid // Event containing the contract specifics and raw log diff --git a/pdp/contract/types.go b/pdp/contract/types.go index bb46e71a3..cf7c9bf89 100644 --- a/pdp/contract/types.go +++ b/pdp/contract/types.go @@ -1,9 +1,6 @@ package contract -import "math/big" - // PieceData matches the Solidity PieceData struct type PieceData struct { - Piece struct{ Data []byte } - RawSize *big.Int + Piece struct{ Data []byte } } diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index e43205111..c56c8df6e 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -143,7 +143,6 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask } if hasEnt { - fmt.Println("Piece cid v2 present in index store") have++ continue } @@ -585,7 +584,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask WHERE m.piece_cid_v2 IS NOT NULL AND m.ddo_v1 IS NOT NULL AND m.ddo_v1 != 'null' - AND m.retrieval_v1->>'announce_payload' = 'true' + AND (m.retrieval_v1->>'announce_payload')::boolean = TRUE AND i.piece_cid IS NULL AND p.id IS NULL AND w.id IS NULL;`) diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index f5ed21882..e4b022216 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -174,7 +174,18 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do } if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { - subPieces = deal.Data.Format.Aggregate.Sub + var found bool + if len(deal.Data.Format.Aggregate.Sub) > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + found = true + } + if len(deal.Data.SourceAggregate.Pieces) > 0 { + subPieces = deal.Data.SourceAggregate.Pieces + found = true + } + if !found { + return false, xerrors.Errorf("no sub pieces for aggregate mk20 deal") + } } } @@ -513,46 +524,41 @@ func IndexAggregate(pieceCid cid.Cid, sectionReader := io.NewSectionReader(reader, int64(strt), int64(leng)) sp := subPieces[j] - //pi := abi.PieceInfo{PieceCID: entry.PieceCID(), Size: abi.PaddedPieceSize(entry.Size)} - //commp, err := commcidv2.CommPFromPieceInfo(pi) - //if err != nil { - // return 0, nil, false, xerrors.Errorf("getting piece commP: %w", err) - //} + if sp.Format.Car != nil { + b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) + if err != nil { + //// Allow one more layer of aggregation to be indexed + //if strings.Contains(err.Error(), "invalid car version") { + // if haveSubPieces { + // if subPieces[j].Car != nil { + // return 0, aggidx, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) + // } + // if subPieces[j].Raw != nil { + // continue + // } + // if subPieces[j].Aggregate != nil { + // b, idx, inter, err = IndexAggregate(commp.PCidV2(), sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) + // if err != nil { + // return totalBlocks, aggidx, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) + // } + // totalBlocks += b + // for k, v := range idx { + // aggidx[k] = append(aggidx[k], v...) + // } + // } + // } else { + // continue + // } + //} + return totalBlocks, aggidx, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) + } - //var idx map[cid.Cid][]datasegment.SegmentDesc + if inter { + return totalBlocks, aggidx, true, nil + } + totalBlocks += b + } - b, inter, err := IndexCAR(sectionReader, bufferSize, recs, addFail) - if err != nil { - //// Allow one more layer of aggregation to be indexed - //if strings.Contains(err.Error(), "invalid car version") { - // if haveSubPieces { - // if subPieces[j].Car != nil { - // return 0, aggidx, false, xerrors.Errorf("invalid car version for subPiece %d: %w", j, err) - // } - // if subPieces[j].Raw != nil { - // continue - // } - // if subPieces[j].Aggregate != nil { - // b, idx, inter, err = IndexAggregate(commp.PCidV2(), sectionReader, abi.PaddedPieceSize(entry.Size), nil, recs, addFail) - // if err != nil { - // return totalBlocks, aggidx, inter, xerrors.Errorf("invalid aggregate for subPiece %d: %w", j, err) - // } - // totalBlocks += b - // for k, v := range idx { - // aggidx[k] = append(aggidx[k], v...) - // } - // } - // } else { - // continue - // } - //} - return totalBlocks, aggidx, false, xerrors.Errorf("indexing subPiece %d: %w", j, err) - } - - if inter { - return totalBlocks, aggidx, true, nil - } - totalBlocks += b aggidx[pieceCid] = append(aggidx[pieceCid], indexstore.Record{ Cid: sp.PieceCID, Offset: strt, diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index 5956d13a2..bdae2ecfd 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -201,8 +201,21 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { - subPieces = deal.Data.Format.Aggregate.Sub + var found bool + if len(deal.Data.Format.Aggregate.Sub) > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + found = true + } + if len(deal.Data.SourceAggregate.Pieces) > 0 { + subPieces = deal.Data.SourceAggregate.Pieces + found = true + } + if !found { + return false, xerrors.Errorf("no sub pieces for aggregate mk20 deal") + } _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.Size, subPieces, recs, addFail) + } else { + return false, xerrors.Errorf("invalid aggregate type") } } diff --git a/tasks/indexing/task_pdp_indexing.go b/tasks/indexing/task_pdp_indexing.go index d44efe159..eb1c72739 100644 --- a/tasks/indexing/task_pdp_indexing.go +++ b/tasks/indexing/task_pdp_indexing.go @@ -101,7 +101,18 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { - subPieces = deal.Data.Format.Aggregate.Sub + var found bool + if len(deal.Data.Format.Aggregate.Sub) > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + found = true + } + if len(deal.Data.SourceAggregate.Pieces) > 0 { + subPieces = deal.Data.SourceAggregate.Pieces + found = true + } + if !found { + return false, xerrors.Errorf("no sub pieces for aggregate PDP deal") + } } } diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go index 367948fe9..ab8025a96 100644 --- a/tasks/indexing/task_pdp_ipni.go +++ b/tasks/indexing/task_pdp_ipni.go @@ -1,8 +1,8 @@ package indexing import ( - "bytes" "context" + "crypto/rand" "errors" "fmt" "net/url" @@ -22,8 +22,6 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -95,8 +93,8 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return true, nil } - var pinfo types.PieceInfo - err = pinfo.UnmarshalCBOR(bytes.NewReader(task.CtxID)) + pinfo := &types.PdpIpniContext{} + err = pinfo.Unmarshal(task.CtxID) if err != nil { return false, xerrors.Errorf("unmarshaling piece info: %w", err) } @@ -157,8 +155,21 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don if deal.Data.Format.Aggregate != nil { if deal.Data.Format.Aggregate.Type > 0 { - subPieces = deal.Data.Format.Aggregate.Sub + var found bool + if len(deal.Data.Format.Aggregate.Sub) > 0 { + subPieces = deal.Data.Format.Aggregate.Sub + found = true + } + if len(deal.Data.SourceAggregate.Pieces) > 0 { + subPieces = deal.Data.SourceAggregate.Pieces + found = true + } + if !found { + return false, xerrors.Errorf("no sub pieces for aggregate mk20 deal") + } _, _, interrupted, err = IndexAggregate(pcid2, reader, pi.PieceInfo().Size, subPieces, recs, addFail) + } else { + return false, xerrors.Errorf("invalid aggregate type") } } @@ -190,7 +201,7 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don } } else { chk := chunker.NewInitialChunker() - err = chk.Accept(pcid2.Hash(), 0, pi.PayloadSize()) + err = chk.Accept(pcid2.Hash(), 0, uint64(pi.PieceInfo().Size)) if err != nil { return false, xerrors.Errorf("adding index to chunk: %w", err) } @@ -332,7 +343,7 @@ func (P *PDPIPNITask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Ta func (P *PDPIPNITask) TypeDetails() harmonytask.TaskTypeDetails { return harmonytask.TaskTypeDetails{ - Name: "PDPIPNI", + Name: "PDPIpni", Cost: resources.Resources{ Cpu: 1, Ram: 1 << 30, @@ -360,29 +371,21 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask stop = true // assume we're done until we find a task to schedule var pendings []struct { - ID string `db:"id"` - PieceCid string `db:"piece_cid"` - Size abi.UnpaddedPieceSize `db:"piece_size"` - PieceCidV2 string `db:"piece_cid_v2"` - Announce bool `db:"announce"` - AnnouncePayload bool `db:"announce_payload"` - IndexingCreatedAt time.Time `db:"indexing_created_at"` - Announced bool `db:"announced"` - AnnouncedPayload bool `db:"announced_payload"` + ID string `db:"id"` + PieceCid string `db:"piece_cid_v2"` + Announce bool `db:"announce"` + AnnouncePayload bool `db:"announce_payload"` + Announced bool `db:"announced"` + AnnouncedPayload bool `db:"announced_payload"` } err := tx.Select(&pendings, `SELECT id, piece_cid_v2, - piece_cid, - piece_size, - raw_size, - indexing, announce, announce_payload, announced, - announced_payload, - indexing_created_at + announced_payload FROM pdp_pipeline WHERE indexed = TRUE AND complete = FALSE @@ -400,7 +403,7 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // Mark deal is complete if: // 1. We don't need to announce anything // 2. Both type of announcements are done - if !(p.Announce && p.AnnouncePayload) || (p.AnnouncePayload && p.AnnouncedPayload) { + if !(p.Announce && p.AnnouncePayload) || (p.Announced && p.AnnouncedPayload) { var n int n, err = tx.Exec(`UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1`, p.ID) @@ -433,18 +436,30 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask return false, xerrors.Errorf("failed to get private libp2p key for PDP: %w", err) } - var pkey []byte + //var pkey []byte - err = tx.QueryRow(`SELECT priv_key FROM eth_keys WHERE role = 'pdp'`).Scan(&pkey) + // TODO: Connect to PDP owner key. Might not be the best approach as keys seem incompatible. + //err = tx.QueryRow(`SELECT private_key FROM eth_keys WHERE role = 'pdp'`).Scan(&pkey) + //if err != nil { + // return false, xerrors.Errorf("failed to get private eth key for PDP: %w", err) + //} + + // generate the ipni provider key + pk, _, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { - return false, xerrors.Errorf("failed to get private eth key for PDP: %w", err) + return false, xerrors.Errorf("failed to generate a new key: %w", err) } - pk, err := crypto.UnmarshalPrivateKey(pkey) + privKey, err = crypto.MarshalPrivateKey(pk) if err != nil { - return false, xerrors.Errorf("unmarshaling private key: %w", err) + return false, xerrors.Errorf("failed to marshal the private key: %w", err) } + //pk, err := crypto.UnmarshalPrivateKey(pkey) + //if err != nil { + // return false, xerrors.Errorf("unmarshaling private key: %w", err) + //} + pid, err := peer.IDFromPublicKey(pk.GetPublic()) if err != nil { return false, xerrors.Errorf("getting peer ID: %w", err) @@ -474,18 +489,17 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // If we need to announce payload and haven't done so, then do it first if p.AnnouncePayload && !p.AnnouncedPayload { - pi := types.PieceInfo{ + pi := &types.PdpIpniContext{ PieceCID: pcid, Payload: true, } - b := new(bytes.Buffer) - err = pi.MarshalCBOR(b) + iContext, err := pi.Marshal() if err != nil { return false, xerrors.Errorf("marshaling piece info: %w", err) } - _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, iContext, false, p.ID, pid.String(), id) if err != nil { if harmonydb.IsErrUniqueContraint(err) { ilog.Infof("Another IPNI announce task already present for piece %s and payload %d in deal %s", p.PieceCid, p.AnnouncePayload, p.ID) @@ -518,17 +532,17 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // If we need to announce piece and haven't done so then do it if p.Announce && !p.Announced { - pi := types.PieceInfo{ + pi := &types.PdpIpniContext{ PieceCID: pcid, Payload: false, } - b := new(bytes.Buffer) - err = pi.MarshalCBOR(b) + + iContext, err := pi.Marshal() if err != nil { return false, xerrors.Errorf("marshaling piece info: %w", err) } - _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, b.Bytes(), false, p.ID, pid.String(), id) + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, iContext, false, p.ID, pid.String(), id) if err != nil { if harmonydb.IsErrUniqueContraint(err) { ilog.Infof("Another IPNI announce task already present for piece %s and payload %d in deal %s", p.PieceCid, p.AnnouncePayload, p.ID) diff --git a/tasks/pdp/dataset_delete_root_watch.go b/tasks/pdp/dataset_delete_root_watch.go index 7e727f681..980ec983e 100644 --- a/tasks/pdp/dataset_delete_root_watch.go +++ b/tasks/pdp/dataset_delete_root_watch.go @@ -130,7 +130,7 @@ func processDataSetPieceDelete(ctx context.Context, db *harmonydb.DB, psd DataSe return false, xerrors.Errorf("failed to delete row from pdp_piece_delete: %w", err) } return true, nil - }) + }, harmonydb.OptionRetry()) if err != nil { return xerrors.Errorf("failed to commit transaction: %w", err) diff --git a/tasks/pdp/task_add_piece.go b/tasks/pdp/task_add_piece.go index 54370c760..6981bf692 100644 --- a/tasks/pdp/task_add_piece.go +++ b/tasks/pdp/task_add_piece.go @@ -19,7 +19,6 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/passcall" - "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/curio/pdp/contract" "github.com/filecoin-project/curio/tasks/message" @@ -71,20 +70,20 @@ func (p *PDPTaskAddPiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) addPiece := addPieces[0] - pcid, err := cid.Parse(addPiece.PieceCid) - if err != nil { - return false, xerrors.Errorf("failed to parse piece cid: %w", err) - } + //pcid, err := cid.Parse(addPiece.PieceCid) + //if err != nil { + // return false, xerrors.Errorf("failed to parse piece cid: %w", err) + //} pcid2, err := cid.Parse(addPiece.PieceCid2) if err != nil { return false, xerrors.Errorf("failed to parse piece cid: %w", err) } - pi, err := mk20.GetPieceInfo(pcid2) - if err != nil { - return false, xerrors.Errorf("failed to get piece info: %w", err) - } + //pi, err := mk20.GetPieceInfo(pcid2) + //if err != nil { + // return false, xerrors.Errorf("failed to get piece info: %w", err) + //} // Prepare the Ethereum transaction data outside the DB transaction // Obtain the ABI of the PDPVerifier contract @@ -93,10 +92,9 @@ func (p *PDPTaskAddPiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) return false, xerrors.Errorf("getting PDPVerifier ABI: %w", err) } - pieceDataArray := []contract.PieceData{ + pieceDataArray := []contract.CidsCid{ { - Piece: struct{ Data []byte }{Data: pcid.Bytes()}, - RawSize: new(big.Int).SetUint64(uint64(pi.Size.Unpadded())), + Data: pcid2.Bytes(), }, } diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index 9c102e3da..e74211eef 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -185,9 +185,11 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b var pieceParked bool comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // TODO: Review this logic for incomplete pieces // Check if we already have the piece, if found then verify access and skip rest of the processing var pid int64 - err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCIDV1.String(), pi.Size).Scan(&pid) + var complete bool + err = tx.QueryRow(`SELECT id, complete FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCIDV1.String(), pi.Size).Scan(&pid, &complete) if err == nil { // If piece exists then check if we can access the data pr, err := a.sc.PieceReader(ctx, storiface.PieceNumber(pid)) @@ -241,7 +243,7 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b defer func() { if failed { _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) - if err != nil { + if ferr != nil { log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) } } @@ -275,6 +277,11 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b return false, fmt.Errorf("failed to delete parked_piece_refs entries: %w", err) } + _, err = tx.Exec(`UPDATE parked_pieces SET complete = true WHERE id = $1 AND complete = FALSE`, parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to mark piece as complete: %w", err) + } + pdp := deal.Products.PDPV1 retv := deal.Products.RetrievalV1 diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index 84583b8b0..a1f2f3fa9 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -232,13 +232,24 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done // If gas used is 0 fee is maximized gasFee := big.NewInt(0) - proofFee, err := pdpVerifier.CalculateProofFee(callOpts, big.NewInt(dataSetID), gasFee) + pdpVerifierRaw := contract.PDPVerifierRaw{Contract: pdpVerifier} + + calcProofFeeResult := make([]any, 1) + err = pdpVerifierRaw.Call(callOpts, &calcProofFeeResult, "calculateProofFee", big.NewInt(dataSetID), gasFee) if err != nil { return false, xerrors.Errorf("failed to calculate proof fee: %w", err) } - // Add 2x buffer for certainty - proofFee = new(big.Int).Mul(proofFee, big.NewInt(3)) + if len(calcProofFeeResult) == 0 { + return false, xerrors.Errorf("failed to calculate proof fee: wrong number of return values") + } + if calcProofFeeResult[0] == nil { + return false, xerrors.Errorf("failed to calculate proof fee: nil return value") + } + if calcProofFeeResult[0].(*big.Int) == nil { + return false, xerrors.Errorf("failed to calculate proof fee: nil *big.Int return value") + } + proofFee := calcProofFeeResult[0].(*big.Int) // Get the sender address for this dataset owner, _, err := pdpVerifier.GetDataSetStorageProvider(callOpts, big.NewInt(dataSetID)) @@ -307,8 +318,8 @@ func (p *ProveTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return true, nil } -func (p *ProveTask) GenerateProofs(ctx context.Context, pdpService *contract.PDPVerifier, dataSetID int64, seed abi.Randomness, numChallenges int) ([]contract.PDPVerifierProof, error) { - proofs := make([]contract.PDPVerifierProof, numChallenges) +func (p *ProveTask) GenerateProofs(ctx context.Context, pdpService *contract.PDPVerifier, dataSetID int64, seed abi.Randomness, numChallenges int) ([]contract.IPDPTypesProof, error) { + proofs := make([]contract.IPDPTypesProof, numChallenges) callOpts := &bind.CallOpts{ Context: ctx, @@ -396,7 +407,7 @@ func padTo32Bytes(b []byte) []byte { return padded } -func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64, challengedLeaf int64) (contract.PDPVerifierProof, error) { +func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64, challengedLeaf int64) (contract.IPDPTypesProof, error) { //const arity = 2 rootChallengeOffset := challengedLeaf * LeafSize @@ -405,20 +416,20 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 err := p.db.QueryRow(context.Background(), `SELECT piece_cid_v2 FROM pdp_dataset_piece WHERE data_set_id = $1 AND root_id = $2`, dataSetID, rootId).Scan(&pieceCid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get root and subroot: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get root and subroot: %w", err) } pcid, err := cid.Parse(pieceCid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to parse piece CID: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to parse piece CID: %w", err) } pi, err := mk20.GetPieceInfo(pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece info: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get piece info: %w", err) } - var out contract.PDPVerifierProof + var out contract.IPDPTypesProof var rootDigest [32]byte // If piece is less than 100 MiB, let's generate proof directly without using cache @@ -426,23 +437,23 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 // Get original file reader reader, _, err := p.cpr.GetSharedPieceReader(ctx, pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece reader: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get piece reader: %w", err) } defer reader.Close() // Build Merkle tree from padded input memTree, err := proof.BuildSha254Memtree(reader, pi.Size.Unpadded()) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to build memtree: %w", err) } log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf) mProof, err := proof.MemtreeProof(memTree, challengedLeaf) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) } - out = contract.PDPVerifierProof{ + out = contract.IPDPTypesProof{ Leaf: mProof.Leaf, Proof: mProof.Proof, } @@ -459,7 +470,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 has, node, err := p.idx.GetPDPNode(ctx, pcid, snapshotNodeIndex) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get node: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get node: %w", err) } if !has { @@ -471,7 +482,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 log.Debugw("proveRoot", "rootChallengeOffset", rootChallengeOffset, "challengedLeaf", challengedLeaf, "layerIdx", layerIdx, "snapshotNodeIndex", snapshotNodeIndex, "node", node) if node.Layer != layerIdx { - return contract.PDPVerifierProof{}, xerrors.Errorf("node layer mismatch: %d != %d", node.Layer, layerIdx) + return contract.IPDPTypesProof{}, xerrors.Errorf("node layer mismatch: %d != %d", node.Layer, layerIdx) } startLeaf := snapshotNodeIndex << layerIdx @@ -485,7 +496,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 // Get original file reader reader, reportedSize, err := p.cpr.GetSharedPieceReader(ctx, pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get reader: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get reader: %w", err) } defer reader.Close() @@ -500,7 +511,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 memtree, err := proof.BuildSha254Memtree(data, subrootSize.Unpadded()) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to build memtree: %w", err) } // Get challenge leaf in subTree @@ -508,19 +519,19 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 subTreeProof, err := proof.MemtreeProof(memtree, subTreeChallenge) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate sub tree proof: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to generate sub tree proof: %w", err) } log.Debugw("subTreeProof", "subrootProof", subTreeProof) // Verify root of proof if subTreeProof.Root != node.Hash { - return contract.PDPVerifierProof{}, xerrors.Errorf("subroot root mismatch: %x != %x", subTreeProof.Root, node.Hash) + return contract.IPDPTypesProof{}, xerrors.Errorf("subroot root mismatch: %x != %x", subTreeProof.Root, node.Hash) } // Fetch full cached layer from DB layerNodes, err := p.idx.GetPDPLayer(ctx, pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get layer nodes: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get layer nodes: %w", err) } // Arrange snapshot layer into a byte array @@ -532,26 +543,26 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 // Create subTree from snapshot to commP (root) mtree, err := proof.BuildSha254MemtreeFromSnapshot(layerBytes) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to build memtree from snapshot: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to build memtree from snapshot: %w", err) } // Generate merkle proof from snapShot node to commP proofs, err := proof.MemtreeProof(mtree, snapshotNodeIndex) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to generate memtree proof: %w", err) } com, err := commcidv2.CommPFromPCidV2(pcid) if err != nil { - return contract.PDPVerifierProof{}, xerrors.Errorf("failed to get piece commitment: %w", err) + return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get piece commitment: %w", err) } // Verify proof with original root if [32]byte(com.Digest()) != proofs.Root { - return contract.PDPVerifierProof{}, xerrors.Errorf("root digest mismatch: %x != %x", com.Digest(), proofs.Root) + return contract.IPDPTypesProof{}, xerrors.Errorf("root digest mismatch: %x != %x", com.Digest(), proofs.Root) } - out = contract.PDPVerifierProof{ + out = contract.IPDPTypesProof{ Leaf: subTreeProof.Leaf, Proof: append(subTreeProof.Proof, proofs.Proof...), } @@ -560,7 +571,7 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 } if !Verify(out, rootDigest, uint64(challengedLeaf)) { - return contract.PDPVerifierProof{}, xerrors.Errorf("proof verification failed") + return contract.IPDPTypesProof{}, xerrors.Errorf("proof verification failed") } // Return the completed proof @@ -604,7 +615,7 @@ func (p *ProveTask) Adder(taskFunc harmonytask.AddTaskFunc) { p.addFunc.Set(taskFunc) } -func Verify(proof contract.PDPVerifierProof, root [32]byte, position uint64) bool { +func Verify(proof contract.IPDPTypesProof, root [32]byte, position uint64) bool { computedHash := proof.Leaf for i := 0; i < len(proof.Proof); i++ { diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index 75a43f1a1..a833ade51 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -196,7 +196,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo } if failed { _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) - if err != nil { + if ferr != nil { log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) } } @@ -226,9 +226,9 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo // Update PoRep pipeline if deal.Products.DDOV1 != nil { var complete bool - err = tx.QueryRow(`SELECT ddo_v1->>'complete' FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) + err = tx.QueryRow(`SELECT (ddo_v1->>'complete')::boolean FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) if err != nil { - return false, fmt.Errorf("getting deal status: %w", err) + return false, fmt.Errorf("getting porep status: %w", err) } if !complete { spid, err := address.IDFromAddress(deal.Products.DDOV1.Provider) @@ -295,50 +295,35 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo // Update PDP pipeline if deal.Products.PDPV1 != nil { var complete bool - err = tx.QueryRow(`SELECT pdp_v1->>'complete' FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) + err = tx.QueryRow(`SELECT (pdp_v1->>'complete')::boolean FROM market_mk20_deal WHERE id = $1`, id.String()).Scan(&complete) if err != nil { - return false, fmt.Errorf("getting deal status: %w", err) + return false, fmt.Errorf("getting pdp status: %w", err) } if !complete { pdp := deal.Products.PDPV1 retv := deal.Products.RetrievalV1 - var newRefID int64 if refIDUsed { err = tx.QueryRow(` INSERT INTO parked_piece_refs (piece_id, data_url, long_term) VALUES ($1, $2, TRUE) RETURNING ref_id - `, parkedPieceID, "/PUT").Scan(&newRefID) + `, parkedPieceID, "/PUT").Scan(&pieceRefID) if err != nil { return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) } - - n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) - if err != nil { - return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) - } - } else { - n, err := tx.Exec(`INSERT INTO pdp_pipeline ( + } + + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, - pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) - if err != nil { - return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) - } + id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) + if err != nil { + return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) } - } } diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index e6cf13d2a..7b1251c08 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -70,7 +70,7 @@ type CurioStorageDealMarket struct { cfg *config.CurioConfig db *harmonydb.DB pin storageingest.Ingester - miners map[string][]address.Address + miners []address.Address api storageMarketAPI MK12Handler *mk12.MK12 MK20Handler *mk20.MK20 @@ -118,10 +118,6 @@ type MK12Pipeline struct { func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg *config.CurioConfig, ethClient *ethclient.Client, si paths.SectorIndex, mapi storageMarketAPI, as *multictladdr.MultiAddressSelector, sc *ffi.SealCalls) *CurioStorageDealMarket { - moduleMap := make(map[string][]address.Address) - moduleMap[mk12Str] = miners - moduleMap[mk20Str] = miners - urls := make(map[string]http.Header) for _, curl := range cfg.Market.StorageMarketConfig.PieceLocator { urls[curl.URL] = curl.Headers @@ -131,7 +127,7 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * cfg: cfg, db: db, api: mapi, - miners: moduleMap, + miners: miners, si: si, urls: urls, as: as, @@ -143,57 +139,49 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { var err error - for module, miners := range d.miners { - if module == mk12Str { - if len(miners) == 0 { - // Do not start the poller if no minerID present - return nil - } - d.MK12Handler, err = mk12.NewMK12Handler(miners, d.db, d.si, d.api, d.cfg, d.as) - if err != nil { - return err - } + if len(d.miners) == 0 { + // Do not start the poller if no minerID present + return nil + } + d.MK12Handler, err = mk12.NewMK12Handler(d.miners, d.db, d.si, d.api, d.cfg, d.as) + if err != nil { + return err + } - if d.MK12Handler != nil { - for _, miner := range miners { - _, err = d.MK12Handler.GetAsk(ctx, miner) - if err != nil { - if strings.Contains(err.Error(), "no ask found") { - if build.BuildType != build.BuildMainnet && build.BuildType != build.BuildCalibnet { - err = d.MK12Handler.SetAsk(ctx, abi.NewTokenAmount(0), abi.NewTokenAmount(0), miner, legacytypes.MinPieceSize(abi.PaddedPieceSize(128)), legacytypes.MaxPieceSize(abi.PaddedPieceSize(8<<20))) - if err != nil { - return xerrors.Errorf("failed to set ask for miner %s: %w", miner, err) - } - } else { - err = d.MK12Handler.SetAsk(ctx, abi.NewTokenAmount(45211226852), abi.NewTokenAmount(0), miner) - if err != nil { - return xerrors.Errorf("failed to set ask for miner %s: %w", miner, err) - } - } - } else { - return xerrors.Errorf("failed to get miner ask %s: %w", miner, err) + if d.MK12Handler != nil { + for _, miner := range d.miners { + _, err = d.MK12Handler.GetAsk(ctx, miner) + if err != nil { + if strings.Contains(err.Error(), "no ask found") { + if build.BuildType != build.BuildMainnet && build.BuildType != build.BuildCalibnet { + err = d.MK12Handler.SetAsk(ctx, abi.NewTokenAmount(0), abi.NewTokenAmount(0), miner, legacytypes.MinPieceSize(abi.PaddedPieceSize(128)), legacytypes.MaxPieceSize(abi.PaddedPieceSize(8<<20))) + if err != nil { + return xerrors.Errorf("failed to set ask for miner %s: %w", miner, err) + } + } else { + err = d.MK12Handler.SetAsk(ctx, abi.NewTokenAmount(45211226852), abi.NewTokenAmount(0), miner) + if err != nil { + return xerrors.Errorf("failed to set ask for miner %s: %w", miner, err) } } + } else { + return xerrors.Errorf("failed to get miner ask %s: %w", miner, err) } } - - if d.cfg.Ingest.DoSnap { - d.pin, err = storageingest.NewPieceIngesterSnap(ctx, d.db, d.api, miners, d.cfg) - } else { - d.pin, err = storageingest.NewPieceIngester(ctx, d.db, d.api, miners, d.cfg) - } - } - if module == mk20Str && d.pin != nil { - if len(miners) == 0 { - return nil - } - d.MK20Handler, err = mk20.NewMK20Handler(miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as, d.sc) - if err != nil { - return err - } } } + d.MK20Handler, err = mk20.NewMK20Handler(d.miners, d.db, d.si, d.api, d.ethClient, d.cfg, d.as, d.sc) + if err != nil { + return err + } + + if d.cfg.Ingest.DoSnap { + d.pin, err = storageingest.NewPieceIngesterSnap(ctx, d.db, d.api, d.miners, d.cfg) + } else { + d.pin, err = storageingest.NewPieceIngester(ctx, d.db, d.api, d.miners, d.cfg) + } + if err != nil { return err } @@ -205,14 +193,8 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { func (d *CurioStorageDealMarket) runPoller(ctx context.Context) { // Start thread to insert mk20 DDO deals into pipeline - for module, miners := range d.miners { - if module == mk20Str { - if len(miners) > 0 { - go d.pipelineInsertLoop(ctx) - go d.migratePieceCIDV2(ctx) - } - } - } + go d.pipelineInsertLoop(ctx) + go d.migratePieceCIDV2(ctx) ticker := time.NewTicker(dealPollerInterval) defer ticker.Stop() @@ -247,14 +229,8 @@ func (d *CurioStorageDealMarket) poll(ctx context.Context) { 5. Once commP is complete, send PSD and find the allocated deal ID 6. Add the deal using pieceIngest */ - for module, miners := range d.miners { - if module == mk12Str { - if len(miners) > 0 { - d.processMK12Deals(ctx) - d.processMK20Deals(ctx) - } - } - } + d.processMK12Deals(ctx) + d.processMK20Deals(ctx) } func (d *CurioStorageDealMarket) processMK12Deals(ctx context.Context) { diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index 5c9089544..d69af5ad7 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -207,6 +207,7 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool var pieceParked bool comm, err := a.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // TODO: Review this logic for pieces which are not complete // Check if we already have the piece, if found then verify access and skip rest of the processing var pid int64 err = tx.QueryRow(`SELECT id FROM parked_pieces WHERE piece_cid = $1 AND piece_padded_size = $2 AND long_term = TRUE`, pi.PieceCIDV1.String(), pi.Size).Scan(&pid) @@ -263,7 +264,7 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool defer func() { if failed { _, ferr := a.db.Exec(ctx, `DELETE FROM parked_piece_refs WHERE ref_id = $1`, pieceRefID) - if err != nil { + if ferr != nil { log.Errorf("failed to delete parked_piece_refs entry: %w", ferr) } } @@ -302,6 +303,11 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool return false, fmt.Errorf("failed to delete parked_piece_refs entries: %w", err) } + _, err = tx.Exec(`UPDATE parked_pieces SET complete = true WHERE id = $1 AND complete = FALSE`, parkedPieceID) + if err != nil { + return false, fmt.Errorf("failed to mark piece as complete: %w", err) + } + var rev mk20.RetrievalV1 if deal.Products.RetrievalV1 != nil { rev = *deal.Products.RetrievalV1 diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_20.go index cf57d0ff6..533d2aca3 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_20.go @@ -83,8 +83,8 @@ func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) d.created_at, d.id, d.piece_cid_v2, - d.ddo_v1->'ddo'->>'provider' AS miner, - d.ddo_v1->>'error' AS error, + (d.ddo_v1->'ddo'->>'provider')::text AS miner, + (d.ddo_v1->>'error')::text AS error, CASE WHEN EXISTS ( SELECT 1 FROM market_mk20_pipeline_waiting w From 066597b8f7ab37d9a6fa3cf2896c48c63c5c59d2 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 21 Aug 2025 23:22:33 +0400 Subject: [PATCH 27/55] piece cleanup, string client --- cmd/curio/tasks/tasks.go | 23 +- cmd/sptool/toolbox_deal_client.go | 174 ++++- documentation/en/curio-cli/sptool.md | 1 + extern/filecoin-ffi | 2 +- go.mod | 2 +- go.sum | 4 +- .../harmonydb/sql/20250505-market_mk20.sql | 157 +++- market/indexstore/cql/0002_piece_index.cql | 5 +- market/indexstore/indexstore.go | 17 +- market/indexstore/indexstore_test.go | 3 + market/ipni/ipni-provider/ipni-provider.go | 13 +- market/mk20/client/client.go | 15 +- market/mk20/http/docs.go | 16 +- market/mk20/http/http.go | 50 +- market/mk20/http/swagger.json | 16 +- market/mk20/http/swagger.yaml | 11 +- market/mk20/mk20.go | 78 +- market/mk20/mk20_upload.go | 26 +- market/mk20/types.go | 6 +- market/mk20/utils.go | 191 ++--- pdp/contract/PDPVerifier.abi | 15 +- pdp/contract/PDPVerifier.json | 2 +- pdp/contract/addresses.go | 2 +- pdp/contract/pdp_verifier.go | 21 +- tasks/gc/pipeline_meta_gc.go | 5 + tasks/gc/task_cleanup_piece.go | 732 ++++++++++++++++++ tasks/indexing/task_check_indexes.go | 9 +- tasks/indexing/task_ipni.go | 137 +++- tasks/indexing/task_pdp_indexing.go | 30 +- tasks/indexing/task_pdp_ipni.go | 199 ++++- tasks/pdp/dataset_add_piece_watch.go | 70 +- tasks/pdp/dataset_delete_root_watch.go | 9 + tasks/pdp/task_add_piece.go | 13 +- tasks/pdp/task_aggregation.go | 33 +- tasks/piece/task_aggregate_chunks.go | 12 +- tasks/storage-market/mk20.go | 18 +- tasks/storage-market/task_aggregation.go | 2 +- web/api/webrpc/market.go | 22 +- web/static/pages/piece/piece-info.mjs | 8 +- 39 files changed, 1732 insertions(+), 417 deletions(-) create mode 100644 tasks/gc/task_cleanup_piece.go diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 0f7204e9e..8c4c81287 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -299,20 +299,22 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan es := getSenderEth() sdeps.EthSender = es - pdp.NewWatcherDataSetCreate(db, must.One(dependencies.EthClient.Val()), chainSched) - pdp.NewWatcherPieceAdd(db, chainSched) + ethClient := must.One(dependencies.EthClient.Val()) + + pdp.NewWatcherDataSetCreate(db, ethClient, chainSched) + pdp.NewWatcherPieceAdd(db, chainSched, ethClient) pdp.NewWatcherDelete(db, chainSched) pdp.NewWatcherPieceDelete(db, chainSched) - pdpProveTask := pdp.NewProveTask(chainSched, db, must.One(dependencies.EthClient.Val()), dependencies.Chain, es, dependencies.CachedPieceReader, iStore) - pdpNextProvingPeriodTask := pdp.NewNextProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) - pdpInitProvingPeriodTask := pdp.NewInitProvingPeriodTask(db, must.One(dependencies.EthClient.Val()), dependencies.Chain, chainSched, es) + pdpProveTask := pdp.NewProveTask(chainSched, db, ethClient, dependencies.Chain, es, dependencies.CachedPieceReader, iStore) + pdpNextProvingPeriodTask := pdp.NewNextProvingPeriodTask(db, ethClient, dependencies.Chain, chainSched, es) + pdpInitProvingPeriodTask := pdp.NewInitProvingPeriodTask(db, ethClient, dependencies.Chain, chainSched, es) pdpNotifTask := pdp.NewPDPNotifyTask(db) - addProofSetTask := pdp.NewPDPTaskAddDataSet(db, es, must.One(dependencies.EthClient.Val()), full) - pdpAddRoot := pdp.NewPDPTaskAddPiece(db, es, must.One(dependencies.EthClient.Val())) - pdpDelRoot := pdp.NewPDPTaskDeletePiece(db, es, must.One(dependencies.EthClient.Val())) - pdpDelProofSetTask := pdp.NewPDPTaskDeleteDataSet(db, es, must.One(dependencies.EthClient.Val()), full) + addProofSetTask := pdp.NewPDPTaskAddDataSet(db, es, ethClient, full) + pdpAddRoot := pdp.NewPDPTaskAddPiece(db, es, ethClient) + pdpDelRoot := pdp.NewPDPTaskDeletePiece(db, es, ethClient) + pdpDelProofSetTask := pdp.NewPDPTaskDeleteDataSet(db, es, ethClient, full) pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) pdpCache := pdp.NewTaskPDPSaveCache(db, dependencies.CachedPieceReader, iStore) @@ -339,6 +341,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan amTask := alertmanager.NewAlertTask(full, db, cfg.Alerting, dependencies.Al) activeTasks = append(activeTasks, amTask) + pcl := gc.NewPieceCleanupTask(db, iStore) + activeTasks = append(activeTasks, pcl) + log.Infow("This Curio instance handles", "miner_addresses", miners, "tasks", lo.Map(activeTasks, func(t harmonytask.TaskInterface, _ int) string { return t.TypeDetails().Name })) diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 9a885a581..ee2ec48f5 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -5,6 +5,8 @@ import ( "bytes" "context" "crypto/rand" + "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -50,6 +52,7 @@ import ( mk12_libp2p "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/mk12" "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/curio/market/mk20/client" lapi "github.com/filecoin-project/lotus/api" chain_types "github.com/filecoin-project/lotus/chain/types" @@ -1721,6 +1724,27 @@ var mk20DealCmd = &cli.Command{ return err } + keyType := client.KeyFromClientAddress(walletAddr) + pkey := walletAddr.Bytes() + ts := time.Now().UTC().Truncate(time.Hour) + msg := sha256.Sum256(bytes.Join([][]byte{pkey, []byte(ts.Format(time.RFC3339))}, []byte{})) + + signature, err := n.Wallet.WalletSign(ctx, walletAddr, msg[:], lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return xerrors.Errorf("signing message: %w", err) + } + + sig, err := signature.MarshalBinary() + if err != nil { + return xerrors.Errorf("marshaling signature: %w", err) + } + + authHeader := fmt.Sprintf("CurioAuth %s:%s:%s", + keyType, + base64.StdEncoding.EncodeToString(pkey), + base64.StdEncoding.EncodeToString(sig), + ) + minfo, err := api.StateMinerInfo(ctx, maddr, chain_types.EmptyTSK) if err != nil { return err @@ -1919,24 +1943,9 @@ var mk20DealCmd = &cli.Command{ } log.Debugw("generated deal id", "id", id) - //msg, err := id.MarshalBinary() - //if err != nil { - // return xerrors.Errorf("failed to marshal deal id: %w", err) - //} - - //sig, err := n.Wallet.WalletSign(ctx, walletAddr, msg, lapi.MsgMeta{Type: lapi.MTDealProposal}) - //if err != nil { - // return xerrors.Errorf("failed to sign deal proposal: %w", err) - //} - - //msgb, err := sig.MarshalBinary() - //if err != nil { - // return xerrors.Errorf("failed to marshal deal proposal signature: %w", err) - //} - deal := mk20.Deal{ Identifier: id, - Client: walletAddr, + Client: walletAddr.String(), Data: &d, Products: p, } @@ -1957,6 +1966,7 @@ var mk20DealCmd = &cli.Command{ return xerrors.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", authHeader) log.Debugw("Headers", "headers", req.Header) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -2029,14 +2039,52 @@ var mk20ClientChunkUploadCmd = &cli.Command{ Usage: "chunk size to be used for the upload", Value: "4 MiB", }, + &cli.StringFlag{ + Name: "wallet", + Usage: "wallet address to be used to initiate the deal", + }, }, Action: func(cctx *cli.Context) error { + ctx := cctx.Context + n, err := Setup(cctx.String(mk12_client_repo.Name)) + if err != nil { + return err + } + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return err + } + + log.Debugw("selected wallet", "wallet", walletAddr) + + keyType := client.KeyFromClientAddress(walletAddr) + pkey := walletAddr.Bytes() + ts := time.Now().UTC().Truncate(time.Hour) + msg := sha256.Sum256(bytes.Join([][]byte{pkey, []byte(ts.Format(time.RFC3339))}, []byte{})) + + signature, err := n.Wallet.WalletSign(ctx, walletAddr, msg[:], lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return xerrors.Errorf("signing message: %w", err) + } + + sig, err := signature.MarshalBinary() + if err != nil { + return xerrors.Errorf("marshaling signature: %w", err) + } + + authHeader := fmt.Sprintf("CurioAuth %s:%s:%s", + keyType, + base64.StdEncoding.EncodeToString(pkey), + base64.StdEncoding.EncodeToString(sig), + ) + if cctx.NArg() != 1 { return xerrors.Errorf("must provide a single file to upload") } + file := cctx.Args().First() log.Debugw("uploading file", "file", file) - ctx := cctx.Context chunkSizeStr := cctx.String("chunk-size") chunkSizem, err := humanize.ParseBytes(chunkSizeStr) @@ -2155,6 +2203,7 @@ var mk20ClientChunkUploadCmd = &cli.Command{ return xerrors.Errorf("failed to upload start create request: %w", err) } client.Header.Set("Content-Type", "application/json") + client.Header.Set("Authorization", authHeader) resp, err := http.DefaultClient.Do(client) if err != nil { return xerrors.Errorf("failed to send request: %w", err) @@ -2175,7 +2224,12 @@ var mk20ClientChunkUploadCmd = &cli.Command{ defer x.Close() for { - resp, err = http.Get(purl.String() + "/market/mk20/uploads/" + dealid.String()) + gc, err := http.NewRequest("GET", purl.String()+"/market/mk20/uploads/"+dealid.String(), nil) + if err != nil { + return xerrors.Errorf("failed to create request: %w", err) + } + gc.Header.Set("Authorization", authHeader) + resp, err := http.DefaultClient.Do(gc) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } @@ -2225,6 +2279,7 @@ var mk20ClientChunkUploadCmd = &cli.Command{ } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Length", fmt.Sprintf("%d", end-start)) + req.Header.Set("Authorization", authHeader) resp, err := http.DefaultClient.Do(req) if err != nil { return xerrors.Errorf("failed to send put request: %w", err) @@ -2242,7 +2297,14 @@ var mk20ClientChunkUploadCmd = &cli.Command{ log.Infow("upload complete") //Finalize the upload - resp, err = http.Post(purl.String()+"/market/mk20/uploads/finalize/"+dealid.String(), "application/json", nil) + pc, err := http.NewRequest("POST", purl.String()+"/market/mk20/uploads/"+dealid.String()+"/finalize", nil) + if err != nil { + return xerrors.Errorf("failed to create finalize request client: %w", err) + } + + pc.Header.Set("Content-Type", "application/json") + pc.Header.Set("Authorization", authHeader) + resp, err = http.DefaultClient.Do(pc) if err != nil { return xerrors.Errorf("failed to send request: %w", err) } @@ -2340,6 +2402,27 @@ var mk20PDPDealCmd = &cli.Command{ log.Debugw("selected wallet", "wallet", walletAddr) + keyType := client.KeyFromClientAddress(walletAddr) + pkey := walletAddr.Bytes() + ts := time.Now().UTC().Truncate(time.Hour) + msg := sha256.Sum256(bytes.Join([][]byte{pkey, []byte(ts.Format(time.RFC3339))}, []byte{})) + + signature, err := n.Wallet.WalletSign(ctx, walletAddr, msg[:], lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return xerrors.Errorf("signing message: %w", err) + } + + sig, err := signature.MarshalBinary() + if err != nil { + return xerrors.Errorf("marshaling signature: %w", err) + } + + authHeader := fmt.Sprintf("CurioAuth %s:%s:%s", + keyType, + base64.StdEncoding.EncodeToString(pkey), + base64.StdEncoding.EncodeToString(sig), + ) + maddr, err := address.NewFromString(cctx.String("provider")) if err != nil { return err @@ -2548,6 +2631,7 @@ var mk20PDPDealCmd = &cli.Command{ ret = &mk20.RetrievalV1{ Indexing: true, AnnouncePayload: true, + AnnouncePiece: true, } } @@ -2597,7 +2681,7 @@ var mk20PDPDealCmd = &cli.Command{ deal := mk20.Deal{ Identifier: id, - Client: walletAddr, + Client: walletAddr.String(), Products: p, } @@ -2621,6 +2705,7 @@ var mk20PDPDealCmd = &cli.Command{ return xerrors.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", authHeader) log.Debugw("Headers", "headers", req.Header) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -2664,12 +2749,45 @@ var mk20ClientUploadCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { + ctx := cctx.Context + n, err := Setup(cctx.String(mk12_client_repo.Name)) + if err != nil { + return err + } + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return err + } + + log.Debugw("selected wallet", "wallet", walletAddr) + + keyType := client.KeyFromClientAddress(walletAddr) + pkey := walletAddr.Bytes() + ts := time.Now().UTC().Truncate(time.Hour) + msg := sha256.Sum256(bytes.Join([][]byte{pkey, []byte(ts.Format(time.RFC3339))}, []byte{})) + + signature, err := n.Wallet.WalletSign(ctx, walletAddr, msg[:], lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return xerrors.Errorf("signing message: %w", err) + } + + sig, err := signature.MarshalBinary() + if err != nil { + return xerrors.Errorf("marshaling signature: %w", err) + } + + authHeader := fmt.Sprintf("CurioAuth %s:%s:%s", + keyType, + base64.StdEncoding.EncodeToString(pkey), + base64.StdEncoding.EncodeToString(sig), + ) + if cctx.NArg() != 1 { return xerrors.Errorf("must provide a single file to upload") } file := cctx.Args().First() log.Debugw("uploading file", "file", file) - ctx := cctx.Context dealid, err := ulid.Parse(cctx.String("deal")) if err != nil { @@ -2757,6 +2875,7 @@ var mk20ClientUploadCmd = &cli.Command{ } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) + req.Header.Set("Authorization", authHeader) resp, err := http.DefaultClient.Do(req) if err != nil { return xerrors.Errorf("failed to send put request: %w", err) @@ -2772,10 +2891,19 @@ var mk20ClientUploadCmd = &cli.Command{ log.Infow("upload complete") //Finalize the upload - resp, err = http.Post(purl.String()+"/market/mk20/upload/"+dealid.String(), "application/json", nil) + req, err = http.NewRequest(http.MethodPost, purl.String()+"/market/mk20/upload/"+dealid.String(), nil) if err != nil { - return xerrors.Errorf("failed to send request: %w", err) + return xerrors.Errorf("failed to create finalize request: %w", err) } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", authHeader) + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("failed to send finalize request: %w", err) + } + if resp.StatusCode != http.StatusOK { respBody, err := io.ReadAll(resp.Body) if err != nil { diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 57b1f3395..619221516 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -1029,5 +1029,6 @@ OPTIONS: --provider value storage provider on-chain address --deal value deal id to upload to --chunk-size value chunk size to be used for the upload (default: "4 MiB") + --wallet value wallet address to be used to initiate the deal --help, -h show help ``` diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 7315a33ed..552ab5c27 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 7315a33ed47f27ef8b224c752268ee73e12cdb19 +Subproject commit 552ab5c27e6bd909f7fbf5c079d0f58b789c3e6f diff --git a/go.mod b/go.mod index bd715ab5d..3435e314b 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-data-segment v0.0.1 github.com/filecoin-project/go-f3 v0.8.9 - github.com/filecoin-project/go-fil-commcid v0.2.0 + github.com/filecoin-project/go-fil-commcid v0.3.1 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 github.com/filecoin-project/go-jsonrpc v0.7.1 github.com/filecoin-project/go-padreader v0.0.1 diff --git a/go.sum b/go.sum index 54b1b02c1..ad8f66c03 100644 --- a/go.sum +++ b/go.sum @@ -325,8 +325,8 @@ github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+ github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM= github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= -github.com/filecoin-project/go-fil-commcid v0.2.0 h1:B+5UX8XGgdg/XsdUpST4pEBviKkFOw+Fvl2bLhSKGpI= -github.com/filecoin-project/go-fil-commcid v0.2.0/go.mod h1:8yigf3JDIil+/WpqR5zoKyP0jBPCOGtEqq/K1CcMy9Q= +github.com/filecoin-project/go-fil-commcid v0.3.1 h1:4EfxpHSlvtkOqa9weG2Yt5kxFmPib2xU7Uc9Lbqk7fs= +github.com/filecoin-project/go-fil-commcid v0.3.1/go.mod h1:z7Ssf8d7kspF9QRAVHDbZ+43JK4mkhbGH5lyph1TnKY= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0/go.mod h1:VH3fAFOru4yyWar4626IoS5+VGE8SfZiBODJLUigEo4= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market_mk20.sql index de6815275..80c3101af 100644 --- a/harmony/harmonydb/sql/20250505-market_mk20.sql +++ b/harmony/harmonydb/sql/20250505-market_mk20.sql @@ -50,6 +50,47 @@ ALTER TABLE ipni ALTER TABLE ipni ADD COLUMN metadata BYTEA NOT NULL DEFAULT '\xa01200'; +-- The order_number column must be completely sequential +ALTER SEQUENCE ipni_order_number_seq CACHE 1; + +-- Add a column in ipni_head to reference a specific ipni row +ALTER TABLE ipni_head + ADD COLUMN head_order_number BIGINT; + +-- Backfill head_order_number to the row you intend as the head. +-- If "head" should point to the latest row with that ad_cid/provider: +WITH latest AS ( + SELECT h.provider, h.head, + MAX(i.order_number) AS order_number + FROM ipni_head h + JOIN ipni i + ON i.provider = h.provider + AND i.ad_cid = h.head + GROUP BY h.provider, h.head +) +UPDATE ipni_head h +SET head_order_number = l.order_number + FROM latest l +WHERE h.provider = l.provider AND h.head = l.head; + +-- Make it NOT NULL once backfilled +ALTER TABLE ipni_head + ALTER COLUMN head_order_number SET NOT NULL; + +-- Switch the FK to reference the unique parent key +ALTER TABLE ipni_head DROP CONSTRAINT ipni_head_head_fkey; + +ALTER TABLE ipni_head + ADD CONSTRAINT ipni_head_head_order_fkey + FOREIGN KEY (head_order_number) + REFERENCES ipni(order_number) + ON DELETE RESTRICT; + +-- Now remove uniqueness on ad_cid (both enforcers). This allows us +-- to chain add/delete/ad/delete for same piece +ALTER TABLE ipni DROP CONSTRAINT ipni_ad_cid_key; +DROP INDEX ipni_ad_cid; + -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). CREATE OR REPLACE FUNCTION process_piece_deal( @@ -180,10 +221,8 @@ CREATE TABLE market_mk20_deal ( created_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), id TEXT PRIMARY KEY, client TEXT NOT NULL, + piece_cid_v2 TEXT, - piece_cid TEXT, -- This is pieceCid V1 to allow easy table lookups - piece_size BIGINT, - raw_size BIGINT, -- For ease data JSONB NOT NULL DEFAULT 'null', @@ -270,12 +309,10 @@ CREATE TABLE market_mk20_download_pipeline ( -- Offline URLs for PoRep deals. CREATE TABLE market_mk20_offline_urls ( id TEXT NOT NULL, - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, + piece_cid_v2 TEXT NOT NULL, url TEXT NOT NULL, headers jsonb NOT NULL DEFAULT '{}', - raw_size BIGINT NOT NULL, - PRIMARY KEY (id, piece_cid, piece_size) + PRIMARY KEY (id, piece_cid_v2) ); -- This table tracks the chunk upload progress for a MK20 deal. Common for both @@ -370,6 +407,7 @@ CREATE TRIGGER trg_ready_at_chunks_update -- locally. This is to allow serving retrievals with piece park. CREATE OR REPLACE FUNCTION process_offline_download( _id TEXT, + _piece_cid_v2 TEXT, _piece_cid TEXT, _piece_size BIGINT, _product TEXT @@ -383,10 +421,10 @@ DECLARE _ref_id BIGINT; BEGIN -- 1. Early exit if no offline match found - SELECT url, headers, raw_size - INTO _url, _headers, _raw_size + SELECT url, headers + INTO _url, _headers FROM market_mk20_offline_urls - WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size; + WHERE id = _id AND piece_cid_v2 = _piece_cid_v2; IF NOT FOUND THEN RETURN FALSE; @@ -396,8 +434,7 @@ BEGIN SELECT deal_aggregation INTO _deal_aggregation FROM market_mk20_pipeline - WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size - LIMIT 1; + WHERE id = _id AND piece_cid_v2 = _piece_cid_v2 LIMIT 1; -- 3. Look for existing piece SELECT id @@ -431,7 +468,7 @@ BEGIN -- 7. Mark the deal as started UPDATE market_mk20_pipeline SET started = TRUE - WHERE id = _id AND piece_cid = _piece_cid AND piece_size = _piece_size AND started = FALSE; + WHERE id = _id AND piece_cid_v2 = _piece_cid_v2 AND started = FALSE; RETURN TRUE; END; @@ -520,9 +557,6 @@ CREATE TABLE pdp_dataset_piece ( client TEXT NOT NULL, piece_cid_v2 TEXT NOT NULL, -- root cid (piececid v2) - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, - raw_size BIGINT NOT NULL, piece BIGINT DEFAULT NULL, -- on-chain index of the piece in the pieceCids sub-array @@ -545,11 +579,8 @@ CREATE TABLE pdp_pipeline ( id TEXT NOT NULL, client TEXT NOT NULL, - piece_cid_v2 TEXT NOT NULL, -- v2 piece_cid - piece_cid TEXT NOT NULL, - piece_size BIGINT NOT NULL, - raw_size BIGINT NOT NULL, + piece_cid_v2 TEXT NOT NULL, -- v2 piece_cid data_set_id BIGINT NOT NULL, @@ -631,8 +662,8 @@ CREATE OR REPLACE FUNCTION insert_pdp_ipni_task( _task_id BIGINT DEFAULT NULL ) RETURNS VOID AS $$ DECLARE -_existing_is_rm BOOLEAN; -_latest_is_rm BOOLEAN; + _existing_is_rm BOOLEAN; + _latest_is_rm BOOLEAN; BEGIN -- Check if ipni_task has the same context_id and provider with a different is_rm value SELECT is_rm INTO _existing_is_rm @@ -642,8 +673,8 @@ BEGIN -- If a different is_rm exists for the same context_id and provider, insert the new task IF FOUND THEN - INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id, created_at, complete) - VALUES (_context_id, _is_rm, _id, _provider, _task_id, TIMEZONE('UTC', NOW()), FALSE); + INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id, created_at) + VALUES (_context_id, _is_rm, _id, _provider, _task_id); RETURN; END IF; @@ -660,8 +691,8 @@ BEGIN END IF; -- If all conditions are met, insert the new task into ipni_task - INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id, created_at, complete) - VALUES (_context_id, _is_rm, _id, _provider, _task_id, TIMEZONE('UTC', NOW()), FALSE); + INSERT INTO pdp_ipni_task (context_id, is_rm, id, provider, task_id) + VALUES (_context_id, _is_rm, _id, _provider, _task_id); END; $$ LANGUAGE plpgsql; @@ -679,7 +710,8 @@ CREATE OR REPLACE FUNCTION insert_ad_and_update_head( _entries TEXT ) RETURNS VOID AS $$ DECLARE -_previous TEXT; + _previous TEXT; + _new_order BIGINT; BEGIN -- Determine the previous ad_cid in the chain for this provider SELECT head INTO _previous @@ -688,13 +720,76 @@ BEGIN -- Insert the new ad into the ipni table with an automatically assigned order_number INSERT INTO ipni (ad_cid, context_id, metadata, is_rm, previous, provider, addresses, signature, entries, piece_cid_v2, piece_cid, piece_size) - VALUES (_ad_cid, _context_id, _metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size); + VALUES (_ad_cid, _context_id, _metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size) RETURNING order_number INTO _new_order; -- Update the ipni_head table to set the new ad as the head of the chain - INSERT INTO ipni_head (provider, head) - VALUES (_provider, _ad_cid) - ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head; + INSERT INTO ipni_head (provider, head, head_order_number) + VALUES (_provider, _ad_cid, _new_order) + ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head, + head_order_number = EXCLUDED.head_order_number; END; $$ LANGUAGE plpgsql; + +CREATE TABLE piece_cleanup ( + id TEXT NOT NULL, + piece_cid_v2 TEXT NOT NULL, + pdp BOOLEAN NOT NULL, + + task_id BIGINT, + + PRIMARY KEY (id, pdp) +); + +-- This functions remove the row from market_piece_deal and then goes on to +-- clean up market_piece_metadata and parked_piece_refs as required +CREATE OR REPLACE FUNCTION remove_piece_deal( + _id TEXT, + _sp_id BIGINT, + _piece_cid TEXT, + _piece_length BIGINT +) RETURNS VOID AS $$ +DECLARE + v_piece_ref BIGINT; + v_remaining BIGINT; +BEGIN + -- 1) Delete the exact deal row and capture piece_ref + DELETE FROM market_piece_deal + WHERE id = _id + AND sp_id = _sp_id + AND piece_cid = _piece_cid + AND piece_length = _piece_length + RETURNING piece_ref + INTO v_piece_ref; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'market_piece_deal not found for id=%, sp_id=%, piece_cid=%, piece_length=%', + _id, _sp_id, _piece_cid, _piece_length; + END IF; + + -- 2) If no other deals reference the same piece, remove metadata + SELECT COUNT(*) + INTO v_remaining + FROM market_piece_deal + WHERE piece_cid = _piece_cid + AND piece_length = _piece_length; + + IF v_remaining = 0 THEN + DELETE FROM market_piece_metadata + WHERE piece_cid = _piece_cid + AND piece_size = _piece_length; + -- (DELETE is idempotent even if no row exists) + END IF; + + -- 3) If present, remove the parked piece reference + IF v_piece_ref IS NOT NULL THEN + DELETE FROM parked_piece_refs + WHERE ref_id = v_piece_ref; + -- (FKs from pdp_* tables will cascade/SET NULL per their definitions) + END IF; +END; +$$ LANGUAGE plpgsql; + + diff --git a/market/indexstore/cql/0002_piece_index.cql b/market/indexstore/cql/0002_piece_index.cql index 91f2ef96d..aa9f87a42 100644 --- a/market/indexstore/cql/0002_piece_index.cql +++ b/market/indexstore/cql/0002_piece_index.cql @@ -1,9 +1,8 @@ CREATE TABLE IF NOT EXISTS PieceToAggregatePiece ( + AggregatePieceCid BLOB PRIMARY KEY, PieceCid BLOB, - AggregatePieceCid BLOB, UnpaddedOffset BIGINT, - UnpaddedLength BIGINT, - PRIMARY KEY (PieceCid, AggregatePieceCid) + UnpaddedLength BIGINT ); CREATE TABLE IF NOT EXISTS PDPCacheLayer ( diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index 7a23b1a25..02e88ab53 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -503,13 +503,18 @@ func (i *IndexStore) FindPieceInAggregate(ctx context.Context, pieceCid cid.Cid) return recs, nil } -func (i *IndexStore) UpdatePieceCidV1ToV2(ctx context.Context, pieceCidV1 cid.Cid, pieceCidV2 cid.Cid) error { - //updateQry := `UPDATE PayloadToPieces SET PieceCid = ? WHERE PieceCid = ?` - //if err := i.session.Query(updateQry, pieceCidV1.Bytes(), pieceCidV2.Bytes()).WithContext(ctx).Exec(); err != nil { - // return xerrors.Errorf("updating piece cid v1 to v2: %w", err) - //} - //return nil +func (i *IndexStore) RemoveAggregateIndex(ctx context.Context, aggregatePieceCid cid.Cid) error { + aggregatePieceCidBytes := aggregatePieceCid.Bytes() + + err := i.session.Query(`DELETE FROM PieceToAggregatePiece WHERE AggregatePieceCid = ?`, aggregatePieceCidBytes).WithContext(ctx).Exec() + if err != nil { + return xerrors.Errorf("deleting aggregate piece cid (P:0x%02x): %w", aggregatePieceCid.Bytes(), err) + } + return nil +} + +func (i *IndexStore) UpdatePieceCidV1ToV2(ctx context.Context, pieceCidV1 cid.Cid, pieceCidV2 cid.Cid) error { p1 := pieceCidV1.Bytes() p2 := pieceCidV2.Bytes() diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index 92caad7e2..ee2881611 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -134,6 +134,9 @@ func TestNewIndexStore(t *testing.T) { require.Len(t, x, 1) require.Equal(t, x[0].Cid, commp.PieceCID) + err = idxStore.RemoveAggregateIndex(ctx, commp.PieceCID) + require.NoError(t, err) + // Drop the tables err = idxStore.session.Query("DROP TABLE PayloadToPieces").Exec() require.NoError(t, err) diff --git a/market/ipni/ipni-provider/ipni-provider.go b/market/ipni/ipni-provider/ipni-provider.go index f02995b47..b16497843 100644 --- a/market/ipni/ipni-provider/ipni-provider.go +++ b/market/ipni/ipni-provider/ipni-provider.go @@ -22,7 +22,6 @@ import ( "github.com/ipni/go-libipni/dagsync/ipnisync/head" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/maurl" - "github.com/ipni/go-libipni/metadata" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -212,6 +211,7 @@ func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (sche Addresses string Signature []byte Entries string + Metadata []byte } err := p.db.Select(ctx, &ads, `SELECT @@ -221,7 +221,8 @@ func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (sche provider, addresses, signature, - entries + entries, + metadata FROM ipni WHERE ad_cid = $1 AND provider = $2`, ad.String(), provider) @@ -245,19 +246,13 @@ func (p *Provider) getAd(ctx context.Context, ad cid.Cid, provider string) (sche return schema.Advertisement{}, xerrors.Errorf("parsing entry CID: %w", err) } - mds := metadata.IpfsGatewayHttp{} - md, err := mds.MarshalBinary() - if err != nil { - return schema.Advertisement{}, xerrors.Errorf("marshalling metadata: %w", err) - } - adv := schema.Advertisement{ Provider: a.Provider, Signature: a.Signature, Entries: cidlink.Link{Cid: e}, ContextID: a.ContextID, IsRm: a.IsRm, - Metadata: md, + Metadata: a.Metadata, } if a.Addresses != "" { diff --git a/market/mk20/client/client.go b/market/mk20/client/client.go index 75521d20b..dc08af9d8 100644 --- a/market/mk20/client/client.go +++ b/market/mk20/client/client.go @@ -187,7 +187,7 @@ func (c *Client) Deal(ctx context.Context, maddr, wallet address.Address, pieceC deal := mk20.Deal{ Identifier: id, - Client: wallet, + Client: wallet.String(), Data: &d, Products: p, } @@ -402,3 +402,16 @@ func (c *Client) DealChunkedUpload(ctx context.Context, dealID string, size, chu } return nil } + +func KeyFromClientAddress(clientAddress address.Address) (key string) { + switch clientAddress.Protocol() { + case address.BLS: + return "bls" + case address.SECP256K1: + return "secp256k1" + case address.Delegated: + return "delegated" + default: + return "" + } +} diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 90f924f25..45b30198a 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -906,7 +906,7 @@ const docTemplate = `{ ] }, "source_httpput": { - "description": "SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import", + "description": "SourceHttpPut allow clients to push piece data after deal is accepted", "allOf": [ { "$ref": "#/definitions/mk20.DataSourceHttpPut" @@ -956,12 +956,8 @@ const docTemplate = `{ "type": "object", "properties": { "client": { - "description": "Client wallet for the deal", - "allOf": [ - { - "$ref": "#/definitions/address.Address" - } - ] + "description": "Client wallet string for the deal", + "type": "string" }, "data": { "description": "Data represents the source of piece data and associated metadata.", @@ -1309,14 +1305,16 @@ const docTemplate = `{ 400, 404, 409, - 500 + 500, + 429 ], "x-enum-varnames": [ "UploadOk", "UploadBadRequest", "UploadNotFound", "UploadChunkAlreadyUploaded", - "UploadServerError" + "UploadServerError", + "UploadRateLimit" ] }, "mk20.UploadStartCode": { diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 59314a08d..16d456b55 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -25,7 +25,6 @@ import ( "github.com/filecoin-project/go-address" - "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/market/mk20" @@ -64,24 +63,19 @@ func dealRateLimitMiddleware() func(http.Handler) http.Handler { return httprate.LimitByIP(50, 1*time.Second) } -func AuthMiddleware(db *harmonydb.DB) func(http.Handler) http.Handler { +func AuthMiddleware(db *harmonydb.DB, cfg *config.CurioConfig) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // TODO: Remove this check once Synapse integration is done - if build.BuildType != build.BuildMainnet { - next.ServeHTTP(w, r) - return - } authHeader := r.Header.Get("Authorization") if authHeader == "" { http.Error(w, "Missing Authorization header", http.StatusUnauthorized) return } - allowed, client, err := mk20.Auth(authHeader, db) + allowed, client, err := mk20.Auth(authHeader, db, cfg) if err != nil { log.Errorw("failed to authenticate request", "err", err) - http.Error(w, err.Error(), http.StatusUnauthorized) + http.Error(w, "Error during authentication: "+err.Error(), http.StatusInternalServerError) return } @@ -138,7 +132,7 @@ func APIRouter(mdh *MK20DealHandler, domainName string) http.Handler { SwaggerInfo.Version = version mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Use(AuthMiddleware(mdh.db)) + mux.Use(AuthMiddleware(mdh.db, mdh.cfg)) mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "request timeout")) mux.Method("GET", "/status/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "request timeout")) mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) @@ -244,7 +238,13 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { log.Infow("received deal proposal", "deal", deal) - result := mdh.dm.MK20Handler.ExecuteDeal(context.Background(), &deal) + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "Missing Authorization header", http.StatusUnauthorized) + return + } + + result := mdh.dm.MK20Handler.ExecuteDeal(context.Background(), &deal, authHeader) log.Infow("deal processed", "id", deal.Identifier, @@ -573,6 +573,12 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re return } + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "Missing Authorization header", http.StatusUnauthorized) + return + } + id, err := ulid.Parse(idStr) if err != nil { log.Errorw("invalid id in url", "id", idStr, "err", err) @@ -591,7 +597,7 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re if len(bytes.TrimSpace(body)) == 0 { log.Debugw("no deal provided, using empty deal to finalize upload", "id", idStr) - mdh.dm.MK20Handler.HandleUploadFinalize(id, nil, w) + mdh.dm.MK20Handler.HandleUploadFinalize(id, nil, w, authHeader) return } @@ -616,7 +622,7 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re return } - mdh.dm.MK20Handler.HandleUploadFinalize(id, &deal, w) + mdh.dm.MK20Handler.HandleUploadFinalize(id, &deal, w, authHeader) } // mk20UpdateDeal handles updating an MK20 deal based on the provided HTTP request. @@ -681,9 +687,15 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques return } + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "Missing Authorization header", http.StatusUnauthorized) + return + } + log.Infow("received deal update proposal", "body", string(body)) - result := mdh.dm.MK20Handler.UpdateDeal(id, &deal) + result := mdh.dm.MK20Handler.UpdateDeal(id, &deal, authHeader) log.Infow("deal updated", "id", deal.Identifier, @@ -759,6 +771,12 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h return } + authHeader := r.Header.Get("Authorization") + if authHeader == "" { + http.Error(w, "Missing Authorization header", http.StatusUnauthorized) + return + } + id, err := ulid.Parse(idStr) if err != nil { log.Errorw("invalid id in url", "id", idStr, "err", err) @@ -777,7 +795,7 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h if len(bytes.TrimSpace(body)) == 0 { log.Debugw("no deal provided, using empty deal to finalize upload", "id", idStr) - mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, nil, w) + mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, nil, w, authHeader) return } @@ -797,5 +815,5 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h return } - mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, &deal, w) + mdh.dm.MK20Handler.HandleSerialUploadFinalize(id, &deal, w, authHeader) } diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index a9fc597a8..3eac8e605 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -897,7 +897,7 @@ ] }, "source_httpput": { - "description": "SourceHTTPPut // allow clients to push piece data after deal accepted, sort of like offline import", + "description": "SourceHttpPut allow clients to push piece data after deal is accepted", "allOf": [ { "$ref": "#/definitions/mk20.DataSourceHttpPut" @@ -947,12 +947,8 @@ "type": "object", "properties": { "client": { - "description": "Client wallet for the deal", - "allOf": [ - { - "$ref": "#/definitions/address.Address" - } - ] + "description": "Client wallet string for the deal", + "type": "string" }, "data": { "description": "Data represents the source of piece data and associated metadata.", @@ -1300,14 +1296,16 @@ 400, 404, 409, - 500 + 500, + 429 ], "x-enum-varnames": [ "UploadOk", "UploadBadRequest", "UploadNotFound", "UploadChunkAlreadyUploaded", - "UploadServerError" + "UploadServerError", + "UploadRateLimit" ] }, "mk20.UploadStartCode": { diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index 6bc350561..fca9d421b 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -94,8 +94,8 @@ definitions: source_httpput: allOf: - $ref: '#/definitions/mk20.DataSourceHttpPut' - description: SourceHTTPPut // allow clients to push piece data after deal - accepted, sort of like offline import + description: SourceHttpPut allow clients to push piece data after deal is + accepted source_offline: allOf: - $ref: '#/definitions/mk20.DataSourceOffline' @@ -124,9 +124,8 @@ definitions: mk20.Deal: properties: client: - allOf: - - $ref: '#/definitions/address.Address' - description: Client wallet for the deal + description: Client wallet string for the deal + type: string data: allOf: - $ref: '#/definitions/mk20.DataSource' @@ -390,6 +389,7 @@ definitions: - 404 - 409 - 500 + - 429 type: integer x-enum-varnames: - UploadOk @@ -397,6 +397,7 @@ definitions: - UploadNotFound - UploadChunkAlreadyUploaded - UploadServerError + - UploadRateLimit mk20.UploadStartCode: enum: - 200 diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index 8cf75df84..d1d22950a 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -104,7 +104,7 @@ func NewMK20Handler(miners []address.Address, db *harmonydb.DB, si paths.SectorI // @Return DealCode // @Return Reason string -func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejectionInfo { +func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal, auth string) *ProviderDealRejectionInfo { defer func() { if r := recover(); r != nil { trace := make([]byte, 1<<16) @@ -115,7 +115,7 @@ func (m *MK20) ExecuteDeal(ctx context.Context, deal *Deal) *ProviderDealRejecti }() // Validate the DataSource - code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20) + code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20, auth) if err != nil { log.Errorw("deal rejected", "deal", deal, "error", err) ret := &ProviderDealRejectionInfo{ @@ -321,7 +321,15 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - alloc, err := m.api.StateGetAllocation(ctx, deal.Client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) + client, err := address.NewFromString(deal.Client) + if err != nil { + return &ProviderDealRejectionInfo{ + HTTPCode: ErrBadProposal, + Reason: "Client address is not valid", + }, nil + } + + alloc, err := m.api.StateGetAllocation(ctx, client, verifreg9.AllocationId(*deal.Products.DDOV1.AllocationId), types.EmptyTSK) if err != nil { return &ProviderDealRejectionInfo{ HTTPCode: ErrServerInternalError, @@ -335,7 +343,7 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe }, nil } - clientID, err := address.IDFromAddress(deal.Client) + clientID, err := address.IDFromAddress(client) if err != nil { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, @@ -449,7 +457,7 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje if pdp.CreateDataSet { n, err := m.DB.Exec(ctx, `INSERT INTO pdp_data_set_create (id, client, record_keeper, extra_data) VALUES ($1, $2, $3, $4)`, - deal.Identifier.String(), deal.Client.String(), pdp.RecordKeeper, pdp.ExtraData) + deal.Identifier.String(), deal.Client, pdp.RecordKeeper, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP proof set create: %w", err) } @@ -460,7 +468,7 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje if pdp.DeleteDataSet { n, err := m.DB.Exec(ctx, `INSERT INTO pdp_data_set_delete (id, client, set_id, extra_data) VALUES ($1, $2, $3, $4)`, - deal.Identifier.String(), deal.Client.String(), *pdp.DataSetID, pdp.ExtraData) + deal.Identifier.String(), deal.Client, *pdp.DataSetID, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP proof set delete: %w", err) } @@ -471,7 +479,7 @@ func (m *MK20) processPDPDeal(ctx context.Context, deal *Deal) *ProviderDealReje if pdp.DeletePiece { n, err := m.DB.Exec(ctx, `INSERT INTO pdp_piece_delete (id, client, set_id, pieces, extra_data) VALUES ($1, $2, $3, $4, $5)`, - deal.Identifier.String(), deal.Client.String(), *pdp.DataSetID, pdp.PieceIDs, pdp.ExtraData) + deal.Identifier.String(), deal.Client, *pdp.DataSetID, pdp.PieceIDs, pdp.ExtraData) if err != nil { return false, xerrors.Errorf("inserting PDP delete root: %w", err) } @@ -532,7 +540,7 @@ func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRe if p.DeleteDataSet || p.AddPiece { pid := *p.DataSetID var exists bool - err := m.DB.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_data_set WHERE id = $1 AND removed = FALSE AND client = $2)`, pid, deal.Client.String()).Scan(&exists) + err := m.DB.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pdp_data_set WHERE id = $1 AND removed = FALSE AND client = $2)`, pid, deal.Client).Scan(&exists) if err != nil { log.Errorw("error checking if proofset exists", "error", err) return &ProviderDealRejectionInfo{ @@ -559,7 +567,7 @@ func (m *MK20) sanitizePDPDeal(ctx context.Context, deal *Deal) (*ProviderDealRe AND r.removed = FALSE AND s.removed = FALSE AND r.client = $3 - AND s.client = $3;`, pid, p.PieceIDs, deal.Client.String()).Scan(&exists) + AND s.client = $3;`, pid, p.PieceIDs, deal.Client).Scan(&exists) if err != nil { log.Errorw("error checking if dataset and pieces exist for the client", "error", err) return &ProviderDealRejectionInfo{ @@ -646,10 +654,9 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error } n, err = tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, deal_aggregation, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`, - dealID, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + id, client, piece_cid_v2, data_set_id, extra_data, deal_aggregation, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + dealID, deal.Client, data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, aggregation, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return xerrors.Errorf("inserting PDP pipeline: %w", err) @@ -744,16 +751,11 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error pBatch := &pgx.Batch{} pBatchSize := 4000 for i, piece := range deal.Data.SourceAggregate.Pieces { - spi, err := GetPieceInfo(piece.PieceCID) - if err != nil { - return xerrors.Errorf("getting piece info: %w", err) - } pBatch.Queue(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, - data_set_id, extra_data, piece_ref, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, - dealID, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), spi.Size, spi.RawSize, - pdp.ExtraData, *pdp.DataSetID, aggregation, i, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) + id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + dealID, deal.Client, piece.PieceCID.String(), pdp.ExtraData, *pdp.DataSetID, + aggregation, i, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if pBatch.Len() > pBatchSize { res := tx.SendBatch(ctx, pBatch) if err := res.Close(); err != nil { @@ -777,18 +779,27 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error func markDownloaded(ctx context.Context, db *harmonydb.DB) { md := func(ctx context.Context, db *harmonydb.DB) { var deals []struct { - ID string `db:"id"` - PieceCID string `db:"piece_cid"` - PieceSize int64 `db:"piece_size"` + ID string `db:"id"` + PieceCID string `db:"piece_cid_v2"` } - err := db.Select(ctx, &deals, `SELECT id, piece_cid, piece_size FROM pdp_pipeline WHERE piece_ref IS NULL`) + err := db.Select(ctx, &deals, `SELECT id, piece_cid_v2 FROM pdp_pipeline WHERE piece_ref IS NULL`) if err != nil { log.Errorw("error getting PDP deals", "error", err) } for _, deal := range deals { _, err = db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + pcid2, err := cid.Decode(deal.PieceCID) + if err != nil { + return false, xerrors.Errorf("decoding piece cid: %w", err) + } + + pi, err := GetPieceInfo(pcid2) + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + var refid int64 err = tx.QueryRow(`SELECT u.ref_id FROM ( SELECT unnest(dp.ref_ids) AS ref_id @@ -798,7 +809,7 @@ func markDownloaded(ctx context.Context, db *harmonydb.DB) { JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id JOIN parked_pieces pp ON pp.id = pr.piece_id WHERE pp.complete = TRUE - LIMIT 1;`, deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1).Scan(&refid) + LIMIT 1;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1).Scan(&refid) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return false, nil @@ -813,22 +824,21 @@ func markDownloaded(ctx context.Context, db *harmonydb.DB) { FROM market_mk20_download_pipeline dp WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 ) - AND ref_id != $5;`, deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1, refid) + AND ref_id != $5;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1, refid) if err != nil { return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) } _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, - deal.ID, deal.PieceCID, deal.PieceSize, ProductNamePDPV1) + deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1) if err != nil { return false, xerrors.Errorf("failed to delete piece from download table: %w", err) } _, err = tx.Exec(`UPDATE pdp_pipeline SET downloaded = TRUE, piece_ref = $1 WHERE id = $2 - AND piece_cid = $3 - AND piece_size = $4`, - refid, deal.ID, deal.PieceCID, deal.PieceSize) + AND piece_cid_v2 = $3`, + refid, deal.ID, deal.PieceCID) if err != nil { return false, xerrors.Errorf("failed to update download statos for PDP pipeline: %w", err) } @@ -858,7 +868,7 @@ func markDownloaded(ctx context.Context, db *harmonydb.DB) { // @Return DealCode // @Return Reason string -func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal) *ProviderDealRejectionInfo { +func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal, auth string) *ProviderDealRejectionInfo { if deal == nil { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, @@ -888,7 +898,7 @@ func (m *MK20) UpdateDeal(id ulid.ULID, deal *Deal) *ProviderDealRejectionInfo { } } - code, nd, np, err := m.updateDealDetails(id, deal) + code, nd, np, err := m.updateDealDetails(id, deal, auth) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) if code == ErrServerInternalError { diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index d2ebb5c29..9cb1952fd 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -14,13 +14,13 @@ import ( "runtime/debug" "time" - "github.com/filecoin-project/go-address" - commcid "github.com/filecoin-project/go-fil-commcid" "github.com/ipfs/go-cid" "github.com/oklog/ulid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + commcid "github.com/filecoin-project/go-fil-commcid" commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-state-types/abi" @@ -442,7 +442,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w // @param deal *Deal [optional] // @Return DealCode -func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { +func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter, auth string) { ctx := context.Background() var exists bool err := m.DB.QueryRow(ctx, `SELECT EXISTS ( @@ -480,7 +480,7 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri if deal != nil { // This is a deal where DataSource was not set - we should update the deal - code, ndeal, _, err := m.updateDealDetails(id, deal) + code, ndeal, _, err := m.updateDealDetails(id, deal, auth) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) if code == ErrServerInternalError { @@ -609,7 +609,7 @@ func (m *MK20) HandleUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWri w.WriteHeader(int(Ok)) } -func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []ProductName, error) { +func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal, auth string) (DealCode, *Deal, []ProductName, error) { ctx := context.Background() // Let's not use request context to avoid DB inconsistencies if deal.Identifier.Compare(id) != 0 { @@ -621,7 +621,7 @@ func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []P } // Validate the deal - code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20) + code, err := deal.Validate(m.DB, &m.cfg.Market.StorageMarketConfig.MK20, auth) if err != nil { return code, nil, nil, err } @@ -640,7 +640,7 @@ func (m *MK20) updateDealDetails(id ulid.ULID, deal *Deal) (DealCode, *Deal, []P } // Get updated deal - ndeal, code, np, err := UpdateDealDetails(ctx, m.DB, id, deal, &m.cfg.Market.StorageMarketConfig.MK20) + ndeal, code, np, err := UpdateDealDetails(ctx, m.DB, id, deal, &m.cfg.Market.StorageMarketConfig.MK20, auth) if err != nil { return code, nil, nil, err } @@ -963,7 +963,7 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW w.WriteHeader(int(UploadOk)) } -func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter) { +func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.ResponseWriter, auth string) { defer func() { if r := recover(); r != nil { trace := make([]byte, 1<<16) @@ -1029,7 +1029,7 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo if deal != nil { // This is a deal where DataSource was not set - we should update the deal - code, ndeal, _, err := m.updateDealDetails(id, deal) + code, ndeal, _, err := m.updateDealDetails(id, deal, auth) if err != nil { log.Errorw("failed to update deal details", "deal", id, "error", err) if code == ErrServerInternalError { @@ -1171,7 +1171,7 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo piece_size, raw_size, url, offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, - id.String(), spid, ddo.ContractAddress, uDeal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + id.String(), spid, ddo.ContractAddress, uDeal.Client, data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), false, retv.Indexing, retv.AnnouncePayload, allocationID, ddo.Duration, aggregation, aggregation) @@ -1202,10 +1202,10 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, + id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, - id.String(), uDeal.Client.String(), uDeal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10)`, + id.String(), uDeal.Client, uDeal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, refID, aggregation, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) diff --git a/market/mk20/types.go b/market/mk20/types.go index 022c6160e..e6eba0e51 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -6,8 +6,6 @@ import ( "github.com/ipfs/go-cid" "github.com/oklog/ulid" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" ) @@ -18,8 +16,8 @@ type Deal struct { // Identifier represents a unique identifier for the deal in UUID format. Identifier ulid.ULID `json:"identifier"` - // Client wallet for the deal - Client address.Address `json:"client"` + // Client wallet string for the deal + Client string `json:"client"` // Data represents the source of piece data and associated metadata. Data *DataSource `json:"data,omitempty"` diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 8c4ae70c8..51356e0c9 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -6,7 +6,6 @@ import ( "crypto/ed25519" "crypto/rand" "crypto/sha256" - "database/sql" "encoding/base64" "encoding/json" "errors" @@ -36,7 +35,7 @@ import ( "github.com/filecoin-project/lotus/lib/sigs" ) -func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { +func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config, Auth string) (DealCode, error) { defer func() { if r := recover(); r != nil { trace := make([]byte, 1<<16) @@ -46,15 +45,11 @@ func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, err } }() - if d.Client.Empty() { - return ErrBadProposal, xerrors.Errorf("no client") + err := validateClient(d.Client, Auth) + if err != nil { + return ErrBadProposal, err } - //code, err := d.ValidateSignature() - //if err != nil { - // return code, xerrors.Errorf("signature validation failed: %w", err) - //} - code, err := d.Products.Validate(db, cfg) if err != nil { return code, xerrors.Errorf("products validation failed: %w", err) @@ -69,33 +64,39 @@ func (d *Deal) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, err return Ok, nil } -//func (d *Deal) ValidateSignature() (DealCode, error) { -// if len(d.Signature) == 0 { -// return ErrBadProposal, xerrors.Errorf("no signature") -// } -// -// sig := &crypto.Signature{} -// err := sig.UnmarshalBinary(d.Signature) -// if err != nil { -// return ErrBadProposal, xerrors.Errorf("invalid signature") -// } -// -// msg, err := d.Identifier.MarshalBinary() -// if err != nil { -// return ErrBadProposal, xerrors.Errorf("invalid identifier") -// } -// -// if sig.Type == crypto.SigTypeBLS || sig.Type == crypto.SigTypeSecp256k1 || sig.Type == crypto.SigTypeDelegated { -// err = sigs.Verify(sig, d.Client, msg) -// if err != nil { -// return ErrBadProposal, xerrors.Errorf("invalid signature") -// } -// return Ok, nil -// } -// -// // Add more types if required in Future -// return ErrBadProposal, xerrors.Errorf("invalid signature type") -//} +func validateClient(client string, auth string) error { + if client == "" { + return xerrors.Errorf("client is empty") + } + + keyType, pubKey, _, err := parseCustomAuth(auth) + if err != nil { + return xerrors.Errorf("parsing auth header: %w", err) + } + + switch keyType { + case "ed25519": + kStr, err := ED25519ToString(pubKey) + if err != nil { + return xerrors.Errorf("invalid public key for auth header: %w", err) + } + if client != kStr { + return xerrors.Errorf("client in deal does not match client in auth header") + } + return nil + case "secp256k1", "bls", "delegated": + addr, err := address.NewFromBytes(pubKey) + if err != nil { + return xerrors.Errorf("invalid public key for auth header: %w", err) + } + if client != addr.String() { + return xerrors.Errorf("client in deal does not match client in auth header") + } + return nil + default: + return fmt.Errorf("unsupported key type: %s", keyType) + } +} func (d DataSource) Validate(db *harmonydb.DB) (DealCode, error) { @@ -414,10 +415,7 @@ type DBPDPV1 struct { type DBDeal struct { Identifier string `db:"id"` Client string `db:"client"` - PieceCIDV2 sql.NullString `db:"piece_cid_v2"` - PieceCID sql.NullString `db:"piece_cid"` - Size sql.NullInt64 `db:"piece_size"` - RawSize sql.NullInt64 `db:"raw_size"` + PieceCIDV2 string `db:"piece_cid_v2"` Data json.RawMessage `db:"data"` DDOv1 json.RawMessage `db:"ddo_v1"` RetrievalV1 json.RawMessage `db:"retrieval_v1"` @@ -427,7 +425,7 @@ type DBDeal struct { func (d *Deal) ToDBDeal() (*DBDeal, error) { ddeal := DBDeal{ Identifier: d.Identifier.String(), - Client: d.Client.String(), + Client: d.Client, } if d.Data != nil { @@ -435,18 +433,7 @@ func (d *Deal) ToDBDeal() (*DBDeal, error) { if err != nil { return nil, fmt.Errorf("marshal data: %w", err) } - commp, err := commcidv2.CommPFromPCidV2(d.Data.PieceCID) - if err != nil { - return nil, fmt.Errorf("invalid piece cid: %w", err) - } - ddeal.PieceCIDV2.String = d.Data.PieceCID.String() - ddeal.PieceCIDV2.Valid = true - ddeal.PieceCID.String = commp.PCidV1().String() - ddeal.PieceCID.Valid = true - ddeal.Size.Int64 = int64(commp.PieceInfo().Size) - ddeal.Size.Valid = true - ddeal.RawSize.Int64 = int64(commp.PayloadSize()) - ddeal.RawSize.Valid = true + ddeal.PieceCIDV2 = d.Data.PieceCID.String() ddeal.Data = dataBytes } else { ddeal.Data = []byte("null") @@ -497,14 +484,19 @@ func (d *Deal) SaveToDB(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } - n, err := tx.Exec(`INSERT INTO market_mk20_deal (id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data, ddo_v1, retrieval_v1, pdp_v1) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + var pieceCid interface{} + + if dbDeal.PieceCIDV2 != "" { + pieceCid = dbDeal.PieceCIDV2 + } else { + pieceCid = nil + } + + n, err := tx.Exec(`INSERT INTO market_mk20_deal (id, client, piece_cid_v2, data, ddo_v1, retrieval_v1, pdp_v1) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, dbDeal.Identifier, dbDeal.Client, - dbDeal.PieceCIDV2, - dbDeal.PieceCID, - dbDeal.Size, - dbDeal.RawSize, + pieceCid, dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, @@ -524,16 +516,20 @@ func (d *Deal) UpdateDealWithTx(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } + var pieceCid interface{} + + if dbDeal.PieceCIDV2 != "" { + pieceCid = dbDeal.PieceCIDV2 + } else { + pieceCid = nil + } + n, err := tx.Exec(`UPDATE market_mk20_deal SET piece_cid_v2 = $1, - piece_cid = $2, - piece_size = $3, - raw_size = $4, - data = $5, - ddo_v1 = $6, - retrieval_v1 = $7, - pdp_v1 = $8`, dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, dbDeal.RawSize, - dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) + data = $2, + ddo_v1 = $3, + retrieval_v1 = $4, + pdp_v1 = $5`, pieceCid, dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) if err != nil { return xerrors.Errorf("update deal: %w", err) } @@ -549,16 +545,20 @@ func (d *Deal) UpdateDeal(tx *harmonydb.Tx) error { return xerrors.Errorf("to db deal: %w", err) } + var pieceCid interface{} + + if dbDeal.PieceCIDV2 != "" { + pieceCid = dbDeal.PieceCIDV2 + } else { + pieceCid = nil + } + n, err := tx.Exec(`UPDATE market_mk20_deal SET piece_cid_v2 = $1, - piece_cid = $2, - piece_size = $3, - raw_size = $4, - data = $5, - ddo_v1 = $6, - retrieval_v1 = $7, - pdp_v1 = $8`, dbDeal.PieceCIDV2, dbDeal.PieceCID, dbDeal.Size, dbDeal.RawSize, - dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) + data = $2, + ddo_v1 = $3, + retrieval_v1 = $4, + pdp_v1 = $5`, pieceCid, dbDeal.Data, dbDeal.DDOv1, dbDeal.RetrievalV1, dbDeal.PDPV1) if err != nil { return xerrors.Errorf("update deal: %w", err) } @@ -645,11 +645,7 @@ func (d *DBDeal) ToDeal() (*Deal, error) { } deal.Identifier = id - client, err := address.NewFromString(d.Client) - if err != nil { - return nil, fmt.Errorf("parse client: %w", err) - } - deal.Client = client + deal.Client = d.Client return &deal, nil } @@ -826,7 +822,7 @@ type UploadStatus struct { MissingChunks []int `json:"missing_chunks"` } -func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config) (*Deal, DealCode, []ProductName, error) { +func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config, auth string) (*Deal, DealCode, []ProductName, error) { ddeal, err := DealFromDB(ctx, db, id) if err != nil { return nil, ErrServerInternalError, nil, xerrors.Errorf("getting deal from DB: %w", err) @@ -863,7 +859,7 @@ func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal newProducts = append(newProducts, ProductNameRetrievalV1) } - code, err := ddeal.Validate(db, cfg) + code, err := ddeal.Validate(db, cfg, auth) if err != nil { return nil, code, nil, xerrors.Errorf("validate deal: %w", err) } @@ -879,9 +875,13 @@ func AuthenticateClient(db *harmonydb.DB, id, client string) (bool, error) { return allowed, nil } -func clientAllowed(ctx context.Context, db *harmonydb.DB, client string) (bool, error) { +func clientAllowed(ctx context.Context, db *harmonydb.DB, client string, cfg *config.CurioConfig) (bool, error) { + if !cfg.Market.StorageMarketConfig.MK20.DenyUnknownClients { + return true, nil + } + var allowed bool - err := db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_clients WHERE client = $1 AND IS allowed)`, client).Scan(&allowed) + err := db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_clients WHERE client = $1 AND allowed = TRUE)`, client).Scan(&allowed) if err != nil { return false, xerrors.Errorf("querying client: %w", err) } @@ -891,12 +891,12 @@ func clientAllowed(ctx context.Context, db *harmonydb.DB, client string) (bool, const Authprefix = "CurioAuth " // Auth verifies the custom authentication header by parsing its contents and validating the signature using the provided database connection. -func Auth(header string, db *harmonydb.DB) (bool, string, error) { +func Auth(header string, db *harmonydb.DB, cfg *config.CurioConfig) (bool, string, error) { keyType, pubKey, sig, err := parseCustomAuth(header) if err != nil { return false, "", xerrors.Errorf("parsing auth header: %w", err) } - return verifySignature(db, keyType, pubKey, sig) + return verifySignature(db, keyType, pubKey, sig, cfg) } func parseCustomAuth(header string) (keyType string, pubKey, sig []byte, err error) { @@ -932,7 +932,7 @@ func parseCustomAuth(header string) (keyType string, pubKey, sig []byte, err err return keyType, pubKey, sig, nil } -func verifySignature(db *harmonydb.DB, keyType string, pubKey, signature []byte) (bool, string, error) { +func verifySignature(db *harmonydb.DB, keyType string, pubKey, signature []byte, cfg *config.CurioConfig) (bool, string, error) { now := time.Now().Truncate(time.Hour) minus1 := now.Add(-59 * time.Minute) plus1 := now.Add(59 * time.Minute) @@ -952,6 +952,15 @@ func verifySignature(db *harmonydb.DB, keyType string, pubKey, signature []byte) if err != nil { return false, "", xerrors.Errorf("invalid ed25519 pubkey: %w", err) } + + allowed, err := clientAllowed(context.Background(), db, keyStr, cfg) + if err != nil { + return false, "", xerrors.Errorf("checking client allowed: %w", err) + } + if !allowed { + return false, "", nil + } + for _, m := range msgs { ok := ed25519.Verify(pubKey, m[:], signature) if ok { @@ -961,13 +970,13 @@ func verifySignature(db *harmonydb.DB, keyType string, pubKey, signature []byte) return false, "", errors.New("invalid ed25519 signature") case "secp256k1", "bls", "delegated": - return verifyFilSignature(db, pubKey, signature, msgs) + return verifyFilSignature(db, pubKey, signature, msgs, cfg) default: return false, "", fmt.Errorf("unsupported key type: %s", keyType) } } -func verifyFilSignature(db *harmonydb.DB, pubKey, signature []byte, msgs [][32]byte) (bool, string, error) { +func verifyFilSignature(db *harmonydb.DB, pubKey, signature []byte, msgs [][32]byte, cfg *config.CurioConfig) (bool, string, error) { signs := &fcrypto.Signature{} err := signs.UnmarshalBinary(signature) if err != nil { @@ -978,12 +987,12 @@ func verifyFilSignature(db *harmonydb.DB, pubKey, signature []byte, msgs [][32]b return false, "", xerrors.Errorf("invalid filecoin pubkey") } - allowed, err := clientAllowed(context.Background(), db, addr.String()) + allowed, err := clientAllowed(context.Background(), db, addr.String(), cfg) if err != nil { return false, "", xerrors.Errorf("checking client allowed: %w", err) } if !allowed { - return false, "", xerrors.Errorf("client not allowed") + return false, "", nil } for _, m := range msgs { diff --git a/pdp/contract/PDPVerifier.abi b/pdp/contract/PDPVerifier.abi index e27d25856..8ef493762 100644 --- a/pdp/contract/PDPVerifier.abi +++ b/pdp/contract/PDPVerifier.abi @@ -1061,6 +1061,19 @@ "type": "uint256[]", "indexed": false, "internalType": "uint256[]" + }, + { + "name": "pieceCids", + "type": "tuple[]", + "indexed": false, + "internalType": "struct Cids.Cid[]", + "components": [ + { + "name": "data", + "type": "bytes", + "internalType": "bytes" + } + ] } ], "anonymous": false @@ -1293,4 +1306,4 @@ } ] } -] +] \ No newline at end of file diff --git a/pdp/contract/PDPVerifier.json b/pdp/contract/PDPVerifier.json index ed32895a5..0700ba02d 100644 --- a/pdp/contract/PDPVerifier.json +++ b/pdp/contract/PDPVerifier.json @@ -1 +1 @@ -{"abi":[{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"newImplementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"errorData","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"addPieces(uint256,(bytes)[],bytes)":"306fc8be","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getScheduledRemovals(uint256)":"6fa44692","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"errorData\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"title\":\"IPDPVerifier\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"notice\":\"Main interface for the PDPVerifier contract\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/interfaces/IPDPVerifier.sol\":\"IPDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts/contracts/utils/Panic.sol\":{\"keccak256\":\"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a\",\"dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG\"]},\"lib/openzeppelin-contracts/contracts/utils/Strings.sol\":{\"keccak256\":\"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57\",\"dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn\"]},\"lib/openzeppelin-contracts/contracts/utils/math/Math.sol\":{\"keccak256\":\"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167\",\"dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol\":{\"keccak256\":\"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8\",\"dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol\":{\"keccak256\":\"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03\",\"dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04\",\"dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d\",\"dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0x9d2999603dc4662015b5d21d4efffbb174cffc67725eca71ff3e967377c201fb\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://292d27f6f6a15c940bc14c59bd3ea7055bf00ed0921fd60af7884d467b60f875\",\"dweb:/ipfs/QmaN9HnGHfhtuyJ5mZoWfrTMpwQ76jRDDhfBAJuuwLbLQ1\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f\",\"dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t\"]},\"src/interfaces/IPDPVerifier.sol\":{\"keccak256\":\"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3\",\"dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"newImplementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"errorData","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct Cids.Cid[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"bytes","name":"","type":"bytes"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/interfaces/IPDPVerifier.sol":"IPDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts/contracts/utils/Panic.sol":{"keccak256":"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a","urls":["bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a","dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Strings.sol":{"keccak256":"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2","urls":["bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57","dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/Math.sol":{"keccak256":"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f","urls":["bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167","dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol":{"keccak256":"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54","urls":["bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8","dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol":{"keccak256":"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3","urls":["bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03","dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ"],"license":"MIT"},"src/BitOps.sol":{"keccak256":"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc","urls":["bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04","dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9","urls":["bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d","dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j"],"license":"Apache-2.0 OR MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0x9d2999603dc4662015b5d21d4efffbb174cffc67725eca71ff3e967377c201fb","urls":["bzz-raw://292d27f6f6a15c940bc14c59bd3ea7055bf00ed0921fd60af7884d467b60f875","dweb:/ipfs/QmaN9HnGHfhtuyJ5mZoWfrTMpwQ76jRDDhfBAJuuwLbLQ1"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846","urls":["bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f","dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t"],"license":"MIT"},"src/interfaces/IPDPVerifier.sol":{"keccak256":"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437","urls":["bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3","dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc"],"license":"MIT"}},"version":1},"id":54} \ No newline at end of file +{"abi":[{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"newImplementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"},{"name":"pieceCids","type":"tuple[]","indexed":false,"internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"errorData","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"addPieces(uint256,(bytes)[],bytes)":"306fc8be","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getScheduledRemovals(uint256)":"6fa44692","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"indexed\":false,\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceCids\",\"type\":\"tuple[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"errorData\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"title\":\"IPDPVerifier\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"notice\":\"Main interface for the PDPVerifier contract\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/interfaces/IPDPVerifier.sol\":\"IPDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts/contracts/utils/Panic.sol\":{\"keccak256\":\"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a\",\"dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG\"]},\"lib/openzeppelin-contracts/contracts/utils/Strings.sol\":{\"keccak256\":\"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57\",\"dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn\"]},\"lib/openzeppelin-contracts/contracts/utils/math/Math.sol\":{\"keccak256\":\"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167\",\"dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol\":{\"keccak256\":\"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8\",\"dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol\":{\"keccak256\":\"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03\",\"dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04\",\"dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d\",\"dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0xc14a7a75b2b3d3be9db007a2314da8b71aa95c44114875dec69a2cdc5a89cdc4\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b037775b1d03dbccbf7eb71e8c8325c9772f1e006a003a186b6b0ebac08355d6\",\"dweb:/ipfs/QmeDeGPaoJ3RfNCFqrTirHZeb8NnExudNpS1kGk2VqQ7vM\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f\",\"dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t\"]},\"src/interfaces/IPDPVerifier.sol\":{\"keccak256\":\"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3\",\"dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"newImplementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false},{"internalType":"struct Cids.Cid[]","name":"pieceCids","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}],"indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"errorData","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct Cids.Cid[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"bytes","name":"","type":"bytes"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/interfaces/IPDPVerifier.sol":"IPDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts/contracts/utils/Panic.sol":{"keccak256":"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a","urls":["bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a","dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Strings.sol":{"keccak256":"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2","urls":["bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57","dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/Math.sol":{"keccak256":"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f","urls":["bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167","dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol":{"keccak256":"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54","urls":["bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8","dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol":{"keccak256":"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3","urls":["bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03","dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ"],"license":"MIT"},"src/BitOps.sol":{"keccak256":"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc","urls":["bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04","dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9","urls":["bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d","dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j"],"license":"Apache-2.0 OR MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0xc14a7a75b2b3d3be9db007a2314da8b71aa95c44114875dec69a2cdc5a89cdc4","urls":["bzz-raw://b037775b1d03dbccbf7eb71e8c8325c9772f1e006a003a186b6b0ebac08355d6","dweb:/ipfs/QmeDeGPaoJ3RfNCFqrTirHZeb8NnExudNpS1kGk2VqQ7vM"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846","urls":["bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f","dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t"],"license":"MIT"},"src/interfaces/IPDPVerifier.sol":{"keccak256":"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437","urls":["bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3","dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc"],"license":"MIT"}},"version":1},"id":53} \ No newline at end of file diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index 37c5c1fac..f6aa46d96 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -13,7 +13,7 @@ import ( const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" -const PDPTestNet = "CHANGEME" +const PDPTestNet = "0x549c257ddb5f9d6fbD195D10eE9e9B14a86D6DB6" type PDPContracts struct { PDPVerifier common.Address diff --git a/pdp/contract/pdp_verifier.go b/pdp/contract/pdp_verifier.go index 170fa9b14..287051c7a 100644 --- a/pdp/contract/pdp_verifier.go +++ b/pdp/contract/pdp_verifier.go @@ -48,7 +48,7 @@ type IPDPTypesProof struct { // PDPVerifierMetaData contains all meta data concerning the PDPVerifier contract. var PDPVerifierMetaData = &bind.MetaData{ - ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE_LOG2\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"activeCount\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"migrate\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"ContractUpgraded\",\"inputs\":[{\"name\":\"version\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":false,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PriceOracleFailure\",\"inputs\":[{\"name\":\"reason\",\"type\":\"bytes\",\"indexed\":false,\"internalType\":\"bytes\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", + ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE_LOG2\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"activeCount\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"migrate\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"ContractUpgraded\",\"inputs\":[{\"name\":\"version\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":false,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"},{\"name\":\"pieceCids\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PriceOracleFailure\",\"inputs\":[{\"name\":\"reason\",\"type\":\"bytes\",\"indexed\":false,\"internalType\":\"bytes\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", } // PDPVerifierABI is the input ABI used to generate the binding from. @@ -2670,14 +2670,15 @@ func (it *PDPVerifierPiecesAddedIterator) Close() error { // PDPVerifierPiecesAdded represents a PiecesAdded event raised by the PDPVerifier contract. type PDPVerifierPiecesAdded struct { - SetId *big.Int - PieceIds []*big.Int - Raw types.Log // Blockchain specific contextual infos + SetId *big.Int + PieceIds []*big.Int + PieceCids []CidsCid + Raw types.Log // Blockchain specific contextual infos } -// FilterPiecesAdded is a free log retrieval operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. +// FilterPiecesAdded is a free log retrieval operation binding the contract event 0x396df50222a87662e94bb7d173792d5e61fe0b193b6ccf791f7ce433f0b28207. // -// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds, (bytes)[] pieceCids) func (_PDPVerifier *PDPVerifierFilterer) FilterPiecesAdded(opts *bind.FilterOpts, setId []*big.Int) (*PDPVerifierPiecesAddedIterator, error) { var setIdRule []interface{} @@ -2692,9 +2693,9 @@ func (_PDPVerifier *PDPVerifierFilterer) FilterPiecesAdded(opts *bind.FilterOpts return &PDPVerifierPiecesAddedIterator{contract: _PDPVerifier.contract, event: "PiecesAdded", logs: logs, sub: sub}, nil } -// WatchPiecesAdded is a free log subscription operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. +// WatchPiecesAdded is a free log subscription operation binding the contract event 0x396df50222a87662e94bb7d173792d5e61fe0b193b6ccf791f7ce433f0b28207. // -// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds, (bytes)[] pieceCids) func (_PDPVerifier *PDPVerifierFilterer) WatchPiecesAdded(opts *bind.WatchOpts, sink chan<- *PDPVerifierPiecesAdded, setId []*big.Int) (event.Subscription, error) { var setIdRule []interface{} @@ -2734,9 +2735,9 @@ func (_PDPVerifier *PDPVerifierFilterer) WatchPiecesAdded(opts *bind.WatchOpts, }), nil } -// ParsePiecesAdded is a log parse operation binding the contract event 0xd9389326b5d4b5a25430b57198f71d0af2a577710c608fb4834f13ca5bfed859. +// ParsePiecesAdded is a log parse operation binding the contract event 0x396df50222a87662e94bb7d173792d5e61fe0b193b6ccf791f7ce433f0b28207. // -// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds) +// Solidity: event PiecesAdded(uint256 indexed setId, uint256[] pieceIds, (bytes)[] pieceCids) func (_PDPVerifier *PDPVerifierFilterer) ParsePiecesAdded(log types.Log) (*PDPVerifierPiecesAdded, error) { event := new(PDPVerifierPiecesAdded) if err := _PDPVerifier.contract.UnpackLog(event, "PiecesAdded", log); err != nil { diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index 0959b185a..094da2953 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -199,6 +199,11 @@ func (s *PipelineGC) cleanupMK20DealPipeline() error { return xerrors.Errorf("failed to clean up sealed deals: %w", err) } + _, err = s.db.Exec(ctx, `DELETE FROM pdp_ipni_task WHERE complete = TRUE;`) + if err != nil { + return xerrors.Errorf("failed to clean up PDP indexing tasks: %w", err) + } + return nil } diff --git a/tasks/gc/task_cleanup_piece.go b/tasks/gc/task_cleanup_piece.go new file mode 100644 index 000000000..7affddd29 --- /dev/null +++ b/tasks/gc/task_cleanup_piece.go @@ -0,0 +1,732 @@ +package gc + +import ( + "bytes" + "context" + "database/sql" + "errors" + "strings" + "time" + + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "github.com/samber/lo" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/promise" + "github.com/filecoin-project/curio/market/indexstore" + "github.com/filecoin-project/curio/market/ipni/types" + "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/curio/tasks/indexing" +) + +type PieceCleanupTask struct { + db *harmonydb.DB + indexStore *indexstore.IndexStore + TF promise.Promise[harmonytask.AddTaskFunc] +} + +func NewPieceCleanupTask(db *harmonydb.DB, indexStore *indexstore.IndexStore) *PieceCleanupTask { + return &PieceCleanupTask{ + db: db, + indexStore: indexStore, + } +} + +func (p *PieceCleanupTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + // TODO: Optimize this Do() as it is currently cumbersome, repetitive and slow. Fix this in a new PR + // TODO: Plug this into PoRep 1.2 and 2.0 clean up as well + // TODO: Remove Deal from MK12 and Mk20? + + ctx := context.Background() + + // To avoid static naming + pdpIpni := indexing.NewPDPIPNITask(nil, nil, nil, nil, taskhelp.Max(0)) + pdpIpniName := pdpIpni.TypeDetails().Name + + poRepIpni := indexing.NewIPNITask(nil, nil, nil, nil, nil, taskhelp.Max(0)) + poRepIpniName := poRepIpni.TypeDetails().Name + + var tasks []struct { + ID string `db:"id"` + PieceCid string `db:"piece_cid_v2"` + PDP bool `db:"pdp"` + } + + err = p.db.Select(ctx, &tasks, `SELECT id, piece_cid_v2, pdp FROM piece_cleanup WHERE task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to get piece cleanup task: %w", err) + } + + if len(tasks) != 1 { + return false, xerrors.Errorf("expected 1 piece cleanup task but got %d", len(tasks)) + } + + task := tasks[0] + + var isMK12 bool + var isMK20 bool + _, err = uuid.Parse(task.ID) + if err == nil { + isMK12 = true + } else { + _, err = ulid.Parse(task.ID) + if err == nil { + isMK20 = true + } + if err != nil { + return false, xerrors.Errorf("failed to parse task ID %s: %w", task.ID, err) + } + } + + pcid2, err := cid.Parse(task.PieceCid) + if err != nil { + return false, xerrors.Errorf("failed to parse piece cid: %w", err) + } + + pi, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return false, xerrors.Errorf("failed to get piece info for piece %s: %w", pcid2, err) + } + + // Did we index this piece? + var indexed bool + err = p.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1 AND piece_size = $2`, pi.PieceCIDV1.String(), pi.Size).Scan(&indexed) + if err != nil { + return false, xerrors.Errorf("failed to check if piece if indexe: %w", err) + } + + dropIndex := true + + type pd struct { + ID string `db:"id"` + SPID int64 `db:"sp_id"` + Sector int64 `db:"sector_num"` + PieceRef sql.NullInt64 `db:"piece_ref"` + } + + var toRM *pd + + var pieceDeals []pd + + // Let's piece deals as we need to make a complicated decision about IPNI and Indexing + err = p.db.Select(ctx, &pieceDeals, `SELECT id, + sp_id, + sector_num, + piece_ref + FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2`, pi.PieceCIDV1.String(), pi.Size) + if err != nil { + return false, xerrors.Errorf("failed to get piece deals: %w", err) + } + + if len(pieceDeals) == 0 { + // This could be due to partial clean up + log.Infof("No piece deals found for piece %s", taskID) + return false, nil + } + /* + Get a list of piece deals + 1. If only single row then check + a) MK1.2 + i) publish IPNI removal ad + ii) Drop index + b) MK2.0 + i) Publish IPNI Ad based on attached product + ii) Drop index if any + iii) Drop Aggregate index + 2. If multiple rows then, check if + a.) MK1.2 + i) If any of the deals is MK1.2 and is not the deal we are cleaning then keep the indexes and don't publish IPNI rm Ad + ii) If there are any MK2.0 deal then check if they are PoRep or PDP + a.) If any of the deals is MK1.2 and is not the deal we are cleaning then keep the indexes + b.) If 2 rows, with same ID then we have PoRep and PDP for same deal. Clean up based on product. + c.) If we have multiple rows with different MK2.0 deals then we need to make a complex decision + i) Check if any of them apart from deal we are cleaning is paying to keep index. If yes, then don't remove them + ii) Check if any of them is paying to keep IPNI payload announced apart from deal we are cleaning. If yes, then don't publish RM ad + iii) Don't publish RM ad for IPNI piece if we have any other PDP deals + */ + + if len(pieceDeals) == 1 { + // Single piece deal, then drop index if deal ID matches + pieceDeal := pieceDeals[0] + if task.ID != pieceDeal.ID { + return false, xerrors.Errorf("piece deal ID %s does not match task ID %s", pieceDeal.ID, task.ID) + } + toRM = &pieceDeal + + UUID, err := uuid.Parse(task.ID) + if err == nil { + pinfo := abi.PieceInfo{ + PieceCID: pi.PieceCIDV1, + Size: pi.Size, + } + b := new(bytes.Buffer) + err = pinfo.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + var peer string + err = tx.QueryRow(`SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, pieceDeal.SPID).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for sp_id %d", pieceDeal.SPID) + } + + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, UUID.String(), pieceDeal.SPID, pieceDeal.Sector, 0, 0, b.Bytes(), true, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, UUID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, UUID) + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, poRepIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } else { + lid, err := ulid.Parse(pieceDeal.ID) + if err == nil { + + deal, err := mk20.DealFromDB(ctx, p.db, lid) + if err != nil { + return false, xerrors.Errorf("failed to get deal for id %s: %w", lid, err) + } + + if deal.Products.RetrievalV1 == nil { + // Return early, we don't need to drop index or publish rm ads + return true, nil + } + + retv := deal.Products.RetrievalV1 + + if task.PDP { + // Let's publish PDP removal first + var peer string + err = p.db.QueryRow(ctx, `SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for PDP provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for PDP") + } + + if retv.AnnouncePiece { + pinfo := types.PdpIpniContext{ + PieceCID: pcid2, + Payload: false, + } + ctxB, err := pinfo.Marshal() + if err != nil { + return false, xerrors.Errorf("failed to marshal pdp context for piece %s: %w", pcid2, err) + } + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, ctxB, true, lid.String(), peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, UUID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, UUID) + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, pdpIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + + } + if retv.AnnouncePayload { + pinfo := types.PdpIpniContext{ + PieceCID: pcid2, + Payload: true, + } + ctxB, err := pinfo.Marshal() + if err != nil { + return false, xerrors.Errorf("failed to marshal pdp context for piece %s: %w", pcid2, err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, ctxB, true, lid.String(), peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, UUID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, UUID) + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, pdpIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } + } else { + // This is a PoRep clean up + pinfo := abi.PieceInfo{ + PieceCID: pi.PieceCIDV1, + Size: pi.Size, + } + b := new(bytes.Buffer) + err = pinfo.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + var peer string + err = p.db.QueryRow(ctx, `SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, pieceDeal.SPID).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for sp_id %d", pieceDeal.SPID) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, lid.String(), pieceDeal.SPID, pieceDeal.Sector, 0, 0, b.Bytes(), true, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, UUID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, UUID) + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, poRepIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } + } else { + return false, xerrors.Errorf("failed to parse piece deal ID %s: %w", pieceDeal.ID, err) + } + } + } else { + // If we have multiple rows + var mk12List []uuid.UUID + var mk20List []ulid.ULID + var pieceDeal pd + for _, pDeal := range pieceDeals { + if pDeal.ID == task.ID { + pieceDeal = pDeal + } + + uid, err := uuid.Parse(pDeal.ID) + if err == nil { + mk12List = append(mk12List, uid) + continue + } + lid, serr := ulid.Parse(pDeal.ID) + if serr == nil { + mk20List = append(mk20List, lid) + continue + } + return false, xerrors.Errorf("failed to parse piece deal ID %s: %w, %w", pieceDeal.ID, err, serr) + + } + toRM = &pieceDeal + lo.Uniq(mk12List) + lo.Uniq(mk20List) + if isMK12 { + rmAccounce := true + if len(mk12List) > 1 { + // Don't drop index or publish removal we have same piece in another deal + dropIndex = false + rmAccounce = false + } + if len(mk20List) > 0 { + for _, d := range mk20List { + deal, err := mk20.DealFromDB(ctx, p.db, d) + if err != nil { + return false, xerrors.Errorf("failed to get deal for id %s: %w", d, err) + } + if deal.Products.RetrievalV1 == nil { + continue + } + retv := deal.Products.RetrievalV1 + if retv.Indexing { + dropIndex = false + } + if retv.AnnouncePayload { + // No need to publish rm Ad as another MK20 deal is paying for it + rmAccounce = false + break + } + } + } + if rmAccounce { + pinfo := abi.PieceInfo{ + PieceCID: pi.PieceCIDV1, + Size: pi.Size, + } + b := new(bytes.Buffer) + err = pinfo.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + var peer string + err = tx.QueryRow(`SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, pieceDeal.SPID).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for sp_id %d", pieceDeal.SPID) + } + + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, pieceDeal.ID, pieceDeal.SPID, pieceDeal.Sector, 0, 0, b.Bytes(), true, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, pieceDeal.ID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, pieceDeal.ID) + return false, nil + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, poRepIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + return true, nil + }) + } + } + + if isMK20 { + rmAccounce := true + rmPiece := true + if len(mk12List) > 1 { + // Don't drop index or publish removal we have same piece in another deal + dropIndex = false + rmAccounce = false + } + if len(mk20List) > 0 { + for _, d := range mk20List { + deal, err := mk20.DealFromDB(ctx, p.db, d) + if err != nil { + return false, xerrors.Errorf("failed to get deal for id %s: %w", d, err) + } + if deal.Products.RetrievalV1 == nil { + continue + } + retv := deal.Products.RetrievalV1 + + // For the deal we are processing + if d.String() == task.ID { + // If we are cleaning up PDP then check PoRep + if task.PDP { + if deal.Products.DDOV1 != nil { + rmAccounce = false + } + } else { + // If we are cleaning up PoRep then check PDP + if deal.Products.PDPV1 != nil { + rmPiece = false + } + if retv.AnnouncePayload { + rmAccounce = false + } + } + } else { + if retv.AnnouncePiece { + rmPiece = false + } + if retv.AnnouncePayload { + rmAccounce = false + } + } + } + } + + if task.PDP { + var peer string + err = p.db.QueryRow(ctx, `SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for PDP provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for PDP") + } + + if rmAccounce { + pinfo := types.PdpIpniContext{ + PieceCID: pcid2, + Payload: true, + } + ctxB, err := pinfo.Marshal() + if err != nil { + return false, xerrors.Errorf("failed to marshal pdp context for piece %s: %w", pcid2, err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, ctxB, true, task.ID, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, task.ID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, task.ID) + return false, nil + } + return false, xerrors.Errorf("failed to publish remove payload ad for piece %s in PDP: %w", pcid2, err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, pdpIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } + + if rmPiece { + pinfo := types.PdpIpniContext{ + PieceCID: pcid2, + Payload: false, + } + ctxB, err := pinfo.Marshal() + if err != nil { + return false, xerrors.Errorf("failed to marshal pdp context for piece %s: %w", pcid2, err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + _, err = tx.Exec(`SELECT insert_pdp_ipni_task($1, $2, $3, $4, $5)`, ctxB, true, task.ID, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, task.ID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, task.ID) + return false, nil + } + return false, xerrors.Errorf("failed to publish remove piece ad for piece %s in PDP: %w", pcid2, err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, pdpIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } + } else { + pinfo := abi.PieceInfo{ + PieceCID: pi.PieceCIDV1, + Size: pi.Size, + } + b := new(bytes.Buffer) + err = pinfo.MarshalCBOR(b) + if err != nil { + return false, xerrors.Errorf("marshaling piece info: %w", err) + } + + p.TF.Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + var peer string + err = tx.QueryRow(`SELECT peer_id FROM ipni_peerid WHERE sp_id = $1`, pieceDeal.SPID).Scan(&peer) + if err != nil { + return false, xerrors.Errorf("failed to get peer id for provider: %w", err) + } + + if peer == "" { + return false, xerrors.Errorf("no peer id found for sp_id %d", pieceDeal.SPID) + } + + _, err = tx.Exec(`SELECT insert_ipni_task($1, $2, $3, $4, $5, $6, $7, $8, $9)`, pieceDeal.ID, pieceDeal.SPID, pieceDeal.Sector, 0, 0, b.Bytes(), true, peer, id) + if err != nil { + if harmonydb.IsErrUniqueContraint(err) { + log.Infof("Another IPNI announce task already present for piece %s in deal %s", pcid2, pieceDeal.ID) + return false, nil + } + if strings.Contains(err.Error(), "already published") { + log.Infof("Piece %s in deal %s is already published", pcid2, pieceDeal.ID) + return false, nil + } + return false, xerrors.Errorf("updating IPNI announcing task id: %w", err) + } + // Fix the harmony_task.name + n, err := tx.Exec(`UPDATE harmony_task SET name = $1 WHERE id = $2`, poRepIpniName, id) + if err != nil { + return false, xerrors.Errorf("failed to update task name: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("failed to update task name: %d rows updated", n) + } + return true, nil + }) + } + } + } + + if dropIndex { + err = dropIndexes(ctx, p.indexStore, pcid2) + if err != nil { + return false, xerrors.Errorf("failed to drop indexes for piece %s: %w", pcid2, err) + } + err = dropAggregateIndex(ctx, p.indexStore, pcid2) + if err != nil { + return false, xerrors.Errorf("failed to drop aggregate index for piece %s: %w", pcid2, err) + } + } + + if task.PDP { + _, err = p.db.Exec(ctx, `SELECT remove_piece_deal($1, $2, $3, $4)`, task.ID, -1, pi.PieceCIDV1.String(), pi.Size) + } else { + _, err = p.db.Exec(ctx, `SELECT remove_piece_deal($1, $2, $3, $4)`, task.ID, toRM.SPID, pi.PieceCIDV1.String(), pi.Size) + } + + if err != nil { + return false, xerrors.Errorf("failed to remove piece deal: %w", err) + } + + _, err = p.db.Exec(ctx, `DELETE FROM piece_cleanup WHERE task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to remove piece cleanup task: %w", err) + } + + return true, nil +} + +func dropIndexes(ctx context.Context, indexStore *indexstore.IndexStore, pieceCid cid.Cid) error { + err := indexStore.RemoveIndexes(ctx, pieceCid) + if err != nil { + return xerrors.Errorf("failed to remove indexes for piece %s: %w", pieceCid, err) + } + return nil +} + +func dropAggregateIndex(ctx context.Context, indexStore *indexstore.IndexStore, pieceCid cid.Cid) error { + err := indexStore.RemoveAggregateIndex(ctx, pieceCid) + if err != nil { + return xerrors.Errorf("failed to remove aggregate index for piece %s: %w", pieceCid, err) + } + return nil +} + +func (p *PieceCleanupTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + return &ids[0], nil +} + +func (p *PieceCleanupTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(50), + Name: "PieceCleanup", + Cost: resources.Resources{ + Cpu: 1, + Ram: 64 << 20, + }, + MaxFailures: 3, + IAmBored: passcall.Every(5*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return p.schedule(context.Background(), taskFunc) + }), + } +} + +func (p *PieceCleanupTask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + var pdp bool + err := tx.QueryRow(`SELECT id, pdp FROM piece_cleanup + WHERE task_id IS NULL + LIMIT 1`).Scan(&did, &pdp) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to query piece_cleanup: %w", err) + } + + _, err = tx.Exec(`UPDATE piece_cleanup SET task_id = $1 WHERE id = $2 AND pdp = $3`, id, did, pdp) + if err != nil { + return false, xerrors.Errorf("failed to update piece_cleanup: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (p *PieceCleanupTask) Adder(taskFunc harmonytask.AddTaskFunc) { + p.TF.Set(taskFunc) +} + +var _ harmonytask.TaskInterface = &PieceCleanupTask{} +var _ = harmonytask.Reg(&PieceCleanupTask{}) diff --git a/tasks/indexing/task_check_indexes.go b/tasks/indexing/task_check_indexes.go index c56c8df6e..c92363cfe 100644 --- a/tasks/indexing/task_check_indexes.go +++ b/tasks/indexing/task_check_indexes.go @@ -330,7 +330,7 @@ func (c *CheckIndexesTask) checkIndexing(ctx context.Context, taskID harmonytask indexing_created_at, complete) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, TRUE, TRUE, TRUE, TRUE, $15, 0, $16, TRUE, NOW(), TRUE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created - deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, int64(pi.RawSize), + deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client, deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, int64(pi.RawSize), false, pieceIDUrl.String(), true, false, ddo.Duration, aggregation, cent.SectorID, cent.PieceOff) if err != nil { @@ -575,8 +575,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask err = c.db.Select(ctx, &ids, `SELECT m.id FROM market_mk20_deal AS m LEFT JOIN ipni AS i - ON m.piece_cid = i.piece_cid - AND m.piece_size = i.piece_size + ON m.piece_cid_v2 = i.piece_cid_v2 LEFT JOIN market_mk20_pipeline AS p ON m.id = p.id LEFT JOIN market_mk20_pipeline_waiting AS w @@ -585,7 +584,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask AND m.ddo_v1 IS NOT NULL AND m.ddo_v1 != 'null' AND (m.retrieval_v1->>'announce_payload')::boolean = TRUE - AND i.piece_cid IS NULL + AND i.piece_cid_v2 IS NULL AND p.id IS NULL AND w.id IS NULL;`) if err != nil { @@ -727,7 +726,7 @@ func (c *CheckIndexesTask) checkIPNIMK20(ctx context.Context, taskID harmonytask indexing_created_at, indexed, complete) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, TRUE, TRUE, TRUE, TRUE, $15, 0, $16, TRUE, NOW(), TRUE, FALSE)`, // We can use reg_seal_proof = 0 because process_piece_deal will prevent new entry from being created - deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pinfo.PieceCIDV1.String(), pinfo.Size, int64(pinfo.RawSize), + deal.Identifier.String(), spid, ddo.ContractAddress, deal.Client, data.PieceCID.String(), pinfo.PieceCIDV1.String(), pinfo.Size, int64(pinfo.RawSize), false, pieceIDUrl.String(), true, true, ddo.Duration, aggregation, src.SectorNum, src.PieceOffset) if err != nil { diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index bdae2ecfd..a0bf65ca2 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -111,6 +111,141 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b return true, nil } + if task.Rm { + comm, err := I.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var ads []struct { + ContextID []byte `db:"context_id"` + IsRm bool `db:"is_rm"` + Previous string `db:"previous"` + Provider string `db:"provider"` + Addresses string `db:"addresses"` + Entries string `db:"entries"` + Metadata []byte `db:"metadata"` + Pcid2 string `db:"piece_cid_v2"` + Pcid1 string `db:"piece_cid"` + Size int64 `db:"piece_size"` + } + + // Get the latest Ad + err = tx.Select(&ads, `SELECT + context_id, + is_rm, + previous, + provider, + addresses, + entries, + metadata, + piece_cid_v2, + piece_cid, + piece_size + FROM ipni + WHERE context_id = $1 + AND provider = $2 + ORDER BY order_number DESC + LIMIT 1`, task.CtxID, task.Prov) + + if err != nil { + return false, xerrors.Errorf("getting ad from DB: %w", err) + } + + if len(ads) == 0 { + return false, xerrors.Errorf("not original ad found for removal ad") + } + + if len(ads) > 1 { + return false, xerrors.Errorf("expected 1 ad but got %d", len(ads)) + } + + a := ads[0] + + e, err := cid.Parse(a.Entries) + if err != nil { + return false, xerrors.Errorf("parsing entry CID: %w", err) + } + + var prev string + + err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("querying previous head: %w", err) + } + + prevCID, err := cid.Parse(prev) + if err != nil { + return false, xerrors.Errorf("parsing previous CID: %w", err) + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM ipni_peerid WHERE sp_id = $1`, task.SPID).Scan(&privKey) + if err != nil { + return false, xerrors.Errorf("failed to get private ipni-libp2p key for PDP: %w", err) + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + adv := schema.Advertisement{ + PreviousID: cidlink.Link{Cid: prevCID}, + Provider: a.Provider, + Addresses: strings.Split(a.Addresses, "|"), + Entries: cidlink.Link{Cid: e}, + ContextID: a.ContextID, + IsRm: true, + Metadata: a.Metadata, + } + + err = adv.Sign(pkey) + if err != nil { + return false, xerrors.Errorf("signing the advertisement: %w", err) + } + + err = adv.Validate() + if err != nil { + return false, xerrors.Errorf("validating the advertisement: %w", err) + } + + adNode, err := adv.ToNode() + if err != nil { + return false, xerrors.Errorf("converting advertisement to node: %w", err) + } + + ad, err := ipniculib.NodeToLink(adNode, schema.Linkproto) + if err != nil { + return false, xerrors.Errorf("converting advertisement to link: %w", err) + } + + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, a.Metadata, a.Pcid2, a.Pcid1, a.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), + adv.Signature, adv.Entries.String()) + + if err != nil { + return false, xerrors.Errorf("adding advertisement to the database: %w", err) + } + + n, err := tx.Exec(`UPDATE ipni_task SET complete = true WHERE task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to mark IPNI task complete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updated %d rows", n) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("store IPNI success: %w", err) + } + + if !comm { + return false, xerrors.Errorf("store IPNI success: failed to commit the transaction") + } + + log.Infow("IPNI task complete", "task_id", taskID) + return true, nil + } + var pi abi.PieceInfo err = pi.UnmarshalCBOR(bytes.NewReader(task.CtxID)) if err != nil { @@ -540,7 +675,7 @@ func (I *IPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFun markComplete = &p.UUID mk20 = p.Mk20 stop = false // we found a sector to work on, keep going - return true, nil + return false, nil } if strings.Contains(err.Error(), "already published") { ilog.Infof("Piece %s in deal %s is already published", p.PieceCid, p.UUID) diff --git a/tasks/indexing/task_pdp_indexing.go b/tasks/indexing/task_pdp_indexing.go index eb1c72739..d8607df47 100644 --- a/tasks/indexing/task_pdp_indexing.go +++ b/tasks/indexing/task_pdp_indexing.go @@ -11,8 +11,6 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/harmonytask" @@ -57,14 +55,11 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) var tasks []struct { ID string `db:"id"` PieceCIDV2 string `db:"piece_cid_v2"` - PieceCID string `db:"piece_cid"` - PieceSize int64 `db:"piece_size"` - RawSize int64 `db:"raw_size"` PieceRef int64 `db:"piece_ref"` Indexing bool `db:"indexing"` } - err = P.db.Select(ctx, &tasks, `SELECT id, piece_cid_v2, piece_cid, piece_size, raw_size, piece_ref, indexing FROM pdp_pipeline WHERE indexing_task_id = $1 AND indexed = FALSE`, taskID) + err = P.db.Select(ctx, &tasks, `SELECT id, piece_cid_v2, piece_ref, indexing FROM pdp_pipeline WHERE indexing_task_id = $1 AND indexed = FALSE`, taskID) if err != nil { return false, xerrors.Errorf("getting PDP pending indexing tasks: %w", err) } @@ -75,17 +70,22 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) task := tasks[0] - var indexed bool - err = P.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1 and piece_size = $2`, task.PieceCID, task.PieceSize).Scan(&indexed) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { - return false, xerrors.Errorf("checking if piece %s is already indexed: %w", task.PieceCIDV2, err) - } - pcid2, err := cid.Parse(task.PieceCIDV2) if err != nil { return false, xerrors.Errorf("parsing piece CID: %w", err) } + pi, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + + var indexed bool + err = P.db.QueryRow(ctx, `SELECT indexed FROM market_piece_metadata WHERE piece_cid = $1 and piece_size = $2`, pi.PieceCIDV1.String(), pi.Size).Scan(&indexed) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("checking if piece %s is already indexed: %w", task.PieceCIDV2, err) + } + id, err := ulid.Parse(task.ID) if err != nil { return false, xerrors.Errorf("parsing task id: %w", err) @@ -121,7 +121,7 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) } if indexed || !task.Indexing || byteData { - err = P.recordCompletion(ctx, taskID, task.ID, task.PieceCID, task.PieceSize, task.RawSize, task.PieceRef, false) + err = P.recordCompletion(ctx, taskID, task.ID, pi.PieceCIDV1.String(), int64(pi.Size), int64(pi.RawSize), task.PieceRef, false) if err != nil { return false, err } @@ -158,7 +158,7 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) var aggidx map[cid.Cid][]indexstore.Record if len(subPieces) > 0 { - blocks, aggidx, interrupted, err = IndexAggregate(pcid2, reader, abi.PaddedPieceSize(task.PieceSize), subPieces, recs, addFail) + blocks, aggidx, interrupted, err = IndexAggregate(pcid2, reader, pi.Size, subPieces, recs, addFail) } else { blocks, interrupted, err = IndexCAR(reader, 4<<20, recs, addFail) } @@ -192,7 +192,7 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) } } - err = P.recordCompletion(ctx, taskID, task.ID, task.PieceCID, task.PieceSize, task.RawSize, task.PieceRef, true) + err = P.recordCompletion(ctx, taskID, task.ID, pi.PieceCIDV1.String(), int64(pi.Size), int64(pi.RawSize), task.PieceRef, true) if err != nil { return false, err } diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go index ab8025a96..33030049e 100644 --- a/tasks/indexing/task_pdp_ipni.go +++ b/tasks/indexing/task_pdp_ipni.go @@ -93,6 +93,141 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return true, nil } + if task.Rm { + comm, err := P.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var ads []struct { + ContextID []byte `db:"context_id"` + IsRm bool `db:"is_rm"` + Previous string `db:"previous"` + Provider string `db:"provider"` + Addresses string `db:"addresses"` + Entries string `db:"entries"` + Metadata []byte `db:"metadata"` + Pcid2 string `db:"piece_cid_v2"` + Pcid1 string `db:"piece_cid"` + Size int64 `db:"piece_size"` + } + + // Get the latest Ad + err = tx.Select(&ads, `SELECT + context_id, + is_rm, + previous, + provider, + addresses, + entries, + metadata, + piece_cid_v2, + piece_cid, + piece_size + FROM ipni + WHERE context_id = $1 + AND provider = $2 + ORDER BY order_number DESC + LIMIT 1`, task.CtxID, task.Prov) + + if err != nil { + return false, xerrors.Errorf("getting ad from DB: %w", err) + } + + if len(ads) == 0 { + return false, xerrors.Errorf("not original ad found for removal ad") + } + + if len(ads) > 1 { + return false, xerrors.Errorf("expected 1 ad but got %d", len(ads)) + } + + a := ads[0] + + e, err := cid.Parse(a.Entries) + if err != nil { + return false, xerrors.Errorf("parsing entry CID: %w", err) + } + + var prev string + + err = tx.QueryRow(`SELECT head FROM ipni_head WHERE provider = $1`, task.Prov).Scan(&prev) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return false, xerrors.Errorf("querying previous head: %w", err) + } + + prevCID, err := cid.Parse(prev) + if err != nil { + return false, xerrors.Errorf("parsing previous CID: %w", err) + } + + var privKey []byte + err = tx.QueryRow(`SELECT priv_key FROM ipni_peerid WHERE sp_id = $1`, -1).Scan(&privKey) + if err != nil { + return false, xerrors.Errorf("failed to get private ipni-libp2p key for PDP: %w", err) + } + + pkey, err := crypto.UnmarshalPrivateKey(privKey) + if err != nil { + return false, xerrors.Errorf("unmarshaling private key: %w", err) + } + + adv := schema.Advertisement{ + PreviousID: cidlink.Link{Cid: prevCID}, + Provider: a.Provider, + Addresses: strings.Split(a.Addresses, "|"), + Entries: cidlink.Link{Cid: e}, + ContextID: a.ContextID, + IsRm: true, + Metadata: a.Metadata, + } + + err = adv.Sign(pkey) + if err != nil { + return false, xerrors.Errorf("signing the advertisement: %w", err) + } + + err = adv.Validate() + if err != nil { + return false, xerrors.Errorf("validating the advertisement: %w", err) + } + + adNode, err := adv.ToNode() + if err != nil { + return false, xerrors.Errorf("converting advertisement to node: %w", err) + } + + ad, err := ipniculib.NodeToLink(adNode, schema.Linkproto) + if err != nil { + return false, xerrors.Errorf("converting advertisement to link: %w", err) + } + + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, + ad.(cidlink.Link).Cid.String(), adv.ContextID, a.Metadata, a.Pcid2, a.Pcid1, a.Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), + adv.Signature, adv.Entries.String()) + + if err != nil { + return false, xerrors.Errorf("adding advertisement to the database: %w", err) + } + + n, err := tx.Exec(`UPDATE pdp_ipni_task SET complete = true WHERE task_id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("failed to mark IPNI task complete: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("updated %d rows", n) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return false, xerrors.Errorf("store IPNI success: %w", err) + } + + if !comm { + return false, xerrors.Errorf("store IPNI success: failed to commit the transaction") + } + + log.Infow("IPNI task complete", "task_id", taskID) + return true, nil + } + pinfo := &types.PdpIpniContext{} err = pinfo.Unmarshal(task.CtxID) if err != nil { @@ -309,7 +444,7 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return false, xerrors.Errorf("converting advertisement to link: %w", err) } - _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + _, err = tx.Exec(`SELECT insert_ad_and_update_head($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, ad.(cidlink.Link).Cid.String(), adv.ContextID, md, pcid2.String(), pi.PieceInfo().PieceCID.String(), pi.PieceInfo().Size, adv.IsRm, adv.Provider, strings.Join(adv.Addresses, "|"), adv.Signature, adv.Entries.String()) @@ -364,8 +499,7 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // schedule submits var stop bool for !stop { - var markComplete *string - var markCompletePayload *string + var markComplete, markCompletePayload, complete *string taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { stop = true // assume we're done until we find a task to schedule @@ -404,28 +538,8 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // 1. We don't need to announce anything // 2. Both type of announcements are done if !(p.Announce && p.AnnouncePayload) || (p.Announced && p.AnnouncedPayload) { - var n int - n, err = tx.Exec(`UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1`, p.ID) - - if err != nil { - return false, xerrors.Errorf("store IPNI success: updating pipeline: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("store IPNI success: updated %d rows", n) - } - - n, err = tx.Exec(`UPDATE market_mk20_deal - SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) - WHERE id = $1;`, p.ID) - if err != nil { - return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) - } - if n != 1 { - return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) - } - - stop = false // we found a task to schedule, keep going - return true, nil + complete = &p.ID + return false, nil } var privKey []byte @@ -517,7 +631,6 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask } stop = false markCompletePayload = &p.ID - // Return early while commiting so we mark complete for payload announcement return true, nil } @@ -562,7 +675,6 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask } stop = false markComplete = &p.ID - // Return early while commiting so we mark complete for piece announcement return true, nil } @@ -597,6 +709,39 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask log.Errorf("store IPNI success: updated %d rows", n) } } + + if complete != nil { + comm, err := P.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + n, err := tx.Exec(`UPDATE pdp_pipeline SET complete = TRUE WHERE id = $1`, *complete) + + if err != nil { + return false, xerrors.Errorf("updating pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 row but updated %d rows", n) + } + + n, err = tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set(pdp_v1, '{complete}', 'true'::jsonb, true) + WHERE id = $1;`, *complete) + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + + stop = false // we found a task to schedule, keep going + ilog.Debugf("Deal %s is marked as complete", *complete) + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("marking deal as complete: %w", err) + } + if !comm { + return xerrors.Errorf("marking deal as complete: failed to commit transaction") + } + } } return nil diff --git a/tasks/pdp/dataset_add_piece_watch.go b/tasks/pdp/dataset_add_piece_watch.go index 5685743a0..7a6cb6a3a 100644 --- a/tasks/pdp/dataset_add_piece_watch.go +++ b/tasks/pdp/dataset_add_piece_watch.go @@ -5,9 +5,10 @@ import ( "encoding/json" "errors" "fmt" - "math/big" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ipfs/go-cid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" @@ -23,9 +24,6 @@ type DataSetPieceAdd struct { ID string `db:"id"` Client string `db:"client"` PieceCID2 string `db:"piece_cid_v2"` // pieceCIDV2 - PieceCID string `db:"piece_cid"` - PieceSize int64 `db:"piece_size"` - RawSize int64 `db:"raw_size"` DataSet uint64 `db:"data_set_id"` PieceRef int64 `db:"piece_ref"` AddMessageHash string `db:"add_message_hash"` @@ -33,9 +31,9 @@ type DataSetPieceAdd struct { } // NewWatcherPieceAdd sets up the watcher for data set piece additions -func NewWatcherPieceAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { +func NewWatcherPieceAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched, ethClient *ethclient.Client) { if err := pcs.AddHandler(func(ctx context.Context, revert, apply *chainTypes.TipSet) error { - err := processPendingDataSetPieceAdds(ctx, db) + err := processPendingDataSetPieceAdds(ctx, db, ethClient) if err != nil { log.Errorf("Failed to process pending data set piece adds: %s", err) } @@ -47,12 +45,12 @@ func NewWatcherPieceAdd(db *harmonydb.DB, pcs *chainsched.CurioChainSched) { } // processPendingDataSetPieceAdds processes piece additions that have been confirmed on-chain -func processPendingDataSetPieceAdds(ctx context.Context, db *harmonydb.DB) error { +func processPendingDataSetPieceAdds(ctx context.Context, db *harmonydb.DB, ethClient *ethclient.Client) error { // Query for pdp_dataset_piece_adds entries where add_message_ok = TRUE var pieceAdds []DataSetPieceAdd err := db.Select(ctx, &pieceAdds, ` - SELECT id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, piece_ref, add_message_hash, add_message_index + SELECT id, client, piece_cid_v2, data_set_id, piece_ref, add_message_hash, add_message_index FROM pdp_pipeline WHERE after_add_piece = TRUE AND after_add_piece_msg = FALSE `) @@ -67,7 +65,7 @@ func processPendingDataSetPieceAdds(ctx context.Context, db *harmonydb.DB) error // Process each piece addition for _, pieceAdd := range pieceAdds { - err := processDataSetPieceAdd(ctx, db, pieceAdd) + err := processDataSetPieceAdd(ctx, db, pieceAdd, ethClient) if err != nil { log.Errorf("Failed to process piece add for tx %s: %s", pieceAdd.AddMessageHash, err) continue @@ -77,7 +75,7 @@ func processPendingDataSetPieceAdds(ctx context.Context, db *harmonydb.DB) error return nil } -func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd DataSetPieceAdd) error { +func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd DataSetPieceAdd, ethClient *ethclient.Client) error { // Retrieve the tx_receipt from message_waits_eth var txReceiptJSON []byte var txSuccess bool @@ -142,8 +140,19 @@ func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd Data } var pieceIds []uint64 + var pieceCids [][]byte eventFound := false + pcid2, err := cid.Parse(pieceAdd.PieceCID2) + if err != nil { + return fmt.Errorf("failed to parse piece CID: %w", err) + } + + parser, err := contract.NewPDPVerifierFilterer(contract.ContractAddresses().PDPVerifier, ethClient) + if err != nil { + return fmt.Errorf("failed to create PDPVerifierFilterer: %w", err) + } + // Iterate over the logs in the receipt for _, vLog := range txReceipt.Logs { // Check if the log corresponds to the PiecesAdded event @@ -151,26 +160,19 @@ func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd Data // The setId is an indexed parameter in Topics[1], but we don't need it here // as we already have the dataset ID from the database - // Parse the non-indexed parameter (pieceIds array) from the data - unpacked, err := event.Inputs.Unpack(vLog.Data) + parsed, err := parser.ParsePiecesAdded(*vLog) if err != nil { - return fmt.Errorf("failed to unpack log data: %w", err) + return fmt.Errorf("failed to parse event log: %w", err) } - // Extract the pieceIds array - if len(unpacked) == 0 { - return fmt.Errorf("no unpacked data found in log") + pieceIds = make([]uint64, len(parsed.PieceIds)) + for i := range parsed.PieceIds { + pieceIds[i] = parsed.PieceIds[i].Uint64() } - // Convert the unpacked pieceIds ([]interface{} containing *big.Int) to []uint64 - bigIntPieceIds, ok := unpacked[0].([]*big.Int) - if !ok { - return fmt.Errorf("failed to convert unpacked data to array") - } - - pieceIds = make([]uint64, len(bigIntPieceIds)) - for i := range bigIntPieceIds { - pieceIds[i] = bigIntPieceIds[i].Uint64() + pieceCids = make([][]byte, len(parsed.PieceCids)) + for i := range parsed.PieceCids { + pieceCids[i] = parsed.PieceCids[i].Data } eventFound = true @@ -184,6 +186,16 @@ func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd Data } pieceId := pieceIds[pieceAdd.AddMessageIndex] + pieceCid := pieceCids[pieceAdd.AddMessageIndex] + + apcid2, err := cid.Cast(pieceCid) + if err != nil { + return fmt.Errorf("failed to cast piece CID: %w", err) + } + + if !apcid2.Equals(pcid2) { + return fmt.Errorf("piece CID in event log does not match piece CID in message") + } // Insert into message_waits_eth and pdp_dataset_pieces comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { @@ -202,23 +214,17 @@ func processDataSetPieceAdd(ctx context.Context, db *harmonydb.DB, pieceAdd Data data_set_id, client, piece_cid_v2, - piece_cid, - piece_size, - raw_size, piece, piece_ref, add_deal_id, add_message_hash, add_message_index ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) `, pieceAdd.DataSet, pieceAdd.Client, pieceAdd.PieceCID2, - pieceAdd.PieceCID, - pieceAdd.PieceSize, - pieceAdd.RawSize, pieceId, pieceAdd.PieceRef, pieceAdd.ID, diff --git a/tasks/pdp/dataset_delete_root_watch.go b/tasks/pdp/dataset_delete_root_watch.go index 980ec983e..b0a012278 100644 --- a/tasks/pdp/dataset_delete_root_watch.go +++ b/tasks/pdp/dataset_delete_root_watch.go @@ -129,6 +129,15 @@ func processDataSetPieceDelete(ctx context.Context, db *harmonydb.DB, psd DataSe if err != nil { return false, xerrors.Errorf("failed to delete row from pdp_piece_delete: %w", err) } + _, err = tx.Exec(`INSERT INTO piece_cleanup (id, piece_cid_v2, pdp) + SELECT p.add_deal_id, p.piece_cid_v2, TRUE + FROM pdp_dataset_piece AS p + WHERE p.data_set_id = $1 + AND p.piece = ANY($2) + ON CONFLICT (id, pdp) DO NOTHING;`, psd.DataSet, psd.Pieces) + if err != nil { + return false, xerrors.Errorf("failed to insert into piece_cleanup: %w", err) + } return true, nil }, harmonydb.OptionRetry()) diff --git a/tasks/pdp/task_add_piece.go b/tasks/pdp/task_add_piece.go index 6981bf692..b22b4b153 100644 --- a/tasks/pdp/task_add_piece.go +++ b/tasks/pdp/task_add_piece.go @@ -48,14 +48,13 @@ func (p *PDPTaskAddPiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) var addPieces []struct { ID string `db:"id"` - PieceCid string `db:"piece_cid"` PieceCid2 string `db:"piece_cid_v2"` DataSetID int64 `db:"data_set_id"` ExtraData []byte `db:"extra_data"` PieceRef string `db:"piece_ref"` } - err = p.db.Select(ctx, &addPieces, `SELECT id, piece_cid, piece_cid_v2, data_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_piece_task_id = $1 AND after_add_piece = FALSE`, taskID) + err = p.db.Select(ctx, &addPieces, `SELECT id, piece_cid_v2, data_set_id, extra_data, piece_ref FROM pdp_pipeline WHERE add_piece_task_id = $1 AND after_add_piece = FALSE`, taskID) if err != nil { return false, xerrors.Errorf("failed to select add piece: %w", err) } @@ -70,21 +69,11 @@ func (p *PDPTaskAddPiece) Do(taskID harmonytask.TaskID, stillOwned func() bool) addPiece := addPieces[0] - //pcid, err := cid.Parse(addPiece.PieceCid) - //if err != nil { - // return false, xerrors.Errorf("failed to parse piece cid: %w", err) - //} - pcid2, err := cid.Parse(addPiece.PieceCid2) if err != nil { return false, xerrors.Errorf("failed to parse piece cid: %w", err) } - //pi, err := mk20.GetPieceInfo(pcid2) - //if err != nil { - // return false, xerrors.Errorf("failed to get piece info: %w", err) - //} - // Prepare the Ethereum transaction data outside the DB transaction // Obtain the ABI of the PDPVerifier contract abiData, err := contract.PDPVerifierMetaData.GetAbi() diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index e74211eef..d62d8d480 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -42,9 +42,7 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b ctx := context.Background() var pieces []struct { - Pcid string `db:"piece_cid"` - Psize int64 `db:"piece_size"` - RawSize int64 `db:"raw_size"` + PieceCidV2 string `db:"piece_cid_v2"` PieceRef int64 `db:"piece_ref"` ID string `db:"id"` AggrIndex int `db:"aggr_index"` @@ -54,9 +52,7 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b err = a.db.Select(ctx, &pieces, ` SELECT - piece_cid, - piece_size, - raw_size, + piece_cid_v2, piece_ref, id, aggr_index, @@ -112,10 +108,10 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b for _, piece := range pieces { if piece.Aggregated { - return false, xerrors.Errorf("piece %s for deal %s already aggregated for task %d", piece.Pcid, piece.ID, taskID) + return false, xerrors.Errorf("piece %s for deal %s already aggregated for task %d", piece.PieceCidV2, piece.ID, taskID) } if piece.Aggregation != 1 { - return false, xerrors.Errorf("incorrect aggregation value for piece %s for deal %s for task %d", piece.Pcid, piece.ID, taskID) + return false, xerrors.Errorf("incorrect aggregation value for piece %s for deal %s for task %d", piece.PieceCidV2, piece.ID, taskID) } if piece.ID != id { return false, xerrors.Errorf("piece details do not match") @@ -148,17 +144,22 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b _ = closer.Close() }() - pcid, err := cid.Parse(piece.Pcid) + pcid2, err := cid.Parse(piece.PieceCidV2) if err != nil { return false, xerrors.Errorf("parsing piece cid: %w", err) } + pinfo, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + pinfos = append(pinfos, abi.PieceInfo{ - Size: abi.PaddedPieceSize(piece.Psize), - PieceCID: pcid, + Size: pinfo.Size, + PieceCID: pinfo.PieceCIDV1, }) - readers = append(readers, io.LimitReader(reader, piece.RawSize)) + readers = append(readers, io.LimitReader(reader, int64(pinfo.RawSize))) refIDs = append(refIDs, piece.PieceRef) } @@ -286,10 +287,10 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b retv := deal.Products.RetrievalV1 n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, + downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10)`, + id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting aggregated piece in PDP pipeline: %w", err) diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index a833ade51..cc8d66224 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -277,7 +277,7 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo piece_size, raw_size, url, offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE)`, - dealID, spid, ddo.ContractAddress, deal.Client.String(), pcid2.String(), pcid.String(), + dealID, spid, ddo.ContractAddress, deal.Client, pcid2.String(), pcid.String(), psize, rawSize, pieceIDUrl.String(), false, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, aggregation, aggregation) @@ -311,12 +311,12 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo return false, fmt.Errorf("failed to create parked_piece_refs entry: %w", err) } } - + n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, + downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10)`, + id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting in PDP pipeline: %w", err) diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index b50486508..7f03d653b 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -264,7 +264,7 @@ func (d *CurioStorageDealMarket) insertDealInPipelineForUpload(ctx context.Conte offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE, TRUE)`, - id, spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), + id, spid, ddo.ContractAddress, deal.Client, deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), false, retv.Indexing, retv.AnnouncePayload, allocationID, ddo.Duration, 0, aggregation) if err != nil { @@ -290,10 +290,10 @@ func (d *CurioStorageDealMarket) insertDealInPipelineForUpload(ctx context.Conte } n, err := tx.Exec(`INSERT INTO pdp_pipeline ( - id, client, piece_cid_v2, piece_cid, piece_size, raw_size, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, TRUE, $10, 0, TRUE, $11, $12, $13)`, - id, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, *pdp.DataSetID, + id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, + downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10)`, + id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { return false, xerrors.Errorf("inserting piece in PDP pipeline: %w", err) @@ -399,7 +399,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation, started) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, TRUE)`, - dealID, spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + dealID, spid, ddo.ContractAddress, deal.Client, data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, false, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, aggregation) if err != nil { @@ -418,7 +418,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, - dealID, spid, ddo.ContractAddress, deal.Client.String(), data.PieceCID.String(), pi.PieceCIDV1.String(), + dealID, spid, ddo.ContractAddress, deal.Client, data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, true, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, aggregation) if err != nil { @@ -526,7 +526,7 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 piece_size, raw_size, offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, aggr_index, started) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)`, - dealID, spid, ddo.ContractAddress, deal.Client.String(), piece.PieceCID.String(), spi.PieceCIDV1.String(), + dealID, spid, ddo.ContractAddress, deal.Client, piece.PieceCID.String(), spi.PieceCIDV1.String(), spi.Size, spi.RawSize, offline, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, 0, data.Format.Aggregate.Type, i, !offline) if pBatch.Len() > pBatchSize { @@ -695,7 +695,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie if piece.Offline && !piece.Downloaded && !piece.Started { comm, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { var updated bool - err = tx.QueryRow(`SELECT process_offline_download($1, $2, $3, $4)`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&updated) + err = tx.QueryRow(`SELECT process_offline_download($1, $2, $3, $4, $5)`, piece.ID, piece.PieceCIDV2, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&updated) if err != nil { if !errors.Is(err, pgx.ErrNoRows) { return false, xerrors.Errorf("failed to start download for offline deal %s: %w", piece.ID, err) diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index d69af5ad7..d146c05ce 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -328,7 +328,7 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool offline, indexing, announce, allocation_id, duration, piece_aggregation, deal_aggregation, started, downloaded, after_commp, aggregated) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, TRUE, TRUE, TRUE, TRUE)`, - id, spid, ddo.ContractAddress, deal.Client.String(), deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), + id, spid, ddo.ContractAddress, deal.Client, deal.Data.PieceCID.String(), pi.PieceCIDV1.String(), pi.Size, pi.RawSize, pieceIDUrl.String(), false, rev.Indexing, rev.AnnouncePayload, allocationID, ddo.Duration, data.Format.Aggregate.Type, data.Format.Aggregate.Type) if err != nil { diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 5318a6774..4b7e4770e 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -576,17 +576,17 @@ func (a *WebRPC) MoveBalanceToEscrow(ctx context.Context, miner string, amount s } type PieceDeal struct { - ID string `db:"id" json:"id"` - BoostDeal bool `db:"boost_deal" json:"boost_deal"` - LegacyDeal bool `db:"legacy_deal" json:"legacy_deal"` - SpId int64 `db:"sp_id" json:"sp_id"` - ChainDealId int64 `db:"chain_deal_id" json:"chain_deal_id"` - Sector int64 `db:"sector_num" json:"sector"` - Offset int64 `db:"piece_offset" json:"offset"` - Length int64 `db:"piece_length" json:"length"` - RawSize int64 `db:"raw_size" json:"raw_size"` - Miner string `json:"miner"` - MK20 bool `db:"-" json:"mk20"` + ID string `db:"id" json:"id"` + BoostDeal bool `db:"boost_deal" json:"boost_deal"` + LegacyDeal bool `db:"legacy_deal" json:"legacy_deal"` + SpId int64 `db:"sp_id" json:"sp_id"` + ChainDealId int64 `db:"chain_deal_id" json:"chain_deal_id"` + Sector int64 `db:"sector_num" json:"sector"` + Offset sql.NullInt64 `db:"piece_offset" json:"offset"` + Length int64 `db:"piece_length" json:"length"` + RawSize int64 `db:"raw_size" json:"raw_size"` + Miner string `json:"miner"` + MK20 bool `db:"-" json:"mk20"` } type PieceInfo struct { diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index 4d7817280..1c4c8d1fa 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -135,7 +135,9 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { - + @@ -216,7 +218,9 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { - + From e2a193e7cdc1437bb626db37dc609bd410b15554 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 1 Sep 2025 18:26:44 -0500 Subject: [PATCH 28/55] ts-client-swagger --- market/mk20/http/swagger.json | 1 + market/mk20/tsclient/.gitignore | 57 + market/mk20/tsclient/Makefile | 33 + market/mk20/tsclient/PROJECT_STRUCTURE.md | 79 + market/mk20/tsclient/README.md | 97 + market/mk20/tsclient/examples/basic-usage.ts | 86 + .../mk20/tsclient/examples/upload-methods.ts | 229 + market/mk20/tsclient/jest.config.js | 17 + market/mk20/tsclient/openapitools.json | 7 + market/mk20/tsclient/package-lock.json | 5837 +++++++++++++++++ market/mk20/tsclient/package.json | 41 + market/mk20/tsclient/scripts/build.sh | 56 + market/mk20/tsclient/src/client.ts | 140 + market/mk20/tsclient/src/index.ts | 36 + market/mk20/tsclient/tests/client.test.ts | 121 + market/mk20/tsclient/tests/setup.ts | 24 + market/mk20/tsclient/tsconfig.json | 27 + 17 files changed, 6888 insertions(+) create mode 100644 market/mk20/tsclient/.gitignore create mode 100644 market/mk20/tsclient/Makefile create mode 100644 market/mk20/tsclient/PROJECT_STRUCTURE.md create mode 100644 market/mk20/tsclient/README.md create mode 100644 market/mk20/tsclient/examples/basic-usage.ts create mode 100644 market/mk20/tsclient/examples/upload-methods.ts create mode 100644 market/mk20/tsclient/jest.config.js create mode 100644 market/mk20/tsclient/openapitools.json create mode 100644 market/mk20/tsclient/package-lock.json create mode 100644 market/mk20/tsclient/package.json create mode 100755 market/mk20/tsclient/scripts/build.sh create mode 100644 market/mk20/tsclient/src/client.ts create mode 100644 market/mk20/tsclient/src/index.ts create mode 100644 market/mk20/tsclient/tests/client.test.ts create mode 100644 market/mk20/tsclient/tests/setup.ts create mode 100644 market/mk20/tsclient/tsconfig.json diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 3eac8e605..b851c219b 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -3,6 +3,7 @@ "info": { "description": "Curio market APIs", "title": "Curio Market 2.0 API", + "version": "2.0.0", "contact": {} }, "paths": { diff --git a/market/mk20/tsclient/.gitignore b/market/mk20/tsclient/.gitignore new file mode 100644 index 000000000..75b8228bf --- /dev/null +++ b/market/mk20/tsclient/.gitignore @@ -0,0 +1,57 @@ +# Dependencies +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Build outputs +dist/ +generated/ + +# TypeScript +*.tsbuildinfo + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +logs +*.log + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory used by tools like istanbul +coverage/ + +# nyc test coverage +.nyc_output + +# Dependency directories +jspm_packages/ + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test diff --git a/market/mk20/tsclient/Makefile b/market/mk20/tsclient/Makefile new file mode 100644 index 000000000..2bed9ba15 --- /dev/null +++ b/market/mk20/tsclient/Makefile @@ -0,0 +1,33 @@ +.PHONY: help install generate compile build clean test + +help: ## Show this help message + @echo 'Usage: make [target]' + @echo '' + @echo 'Targets:' + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +install: ## Install dependencies + npm install + +generate: ## Generate TypeScript client from swagger files + npm run generate + +compile: ## Compile TypeScript to JavaScript + npm run compile + +build: ## Build everything (generate + compile) + npm run build + +clean: ## Clean build artifacts + npm run clean + +test: ## Run tests (placeholder for future test setup) + @echo "Tests not yet implemented" + +dev: ## Development mode - watch for changes and rebuild + @echo "Starting development mode..." + @echo "Run 'make build' to rebuild after changes" + +.PHONY: setup +setup: install generate compile ## Initial setup: install deps, generate client, and compile + @echo "Setup complete! Run 'make build' to rebuild in the future." diff --git a/market/mk20/tsclient/PROJECT_STRUCTURE.md b/market/mk20/tsclient/PROJECT_STRUCTURE.md new file mode 100644 index 000000000..a44ead9e9 --- /dev/null +++ b/market/mk20/tsclient/PROJECT_STRUCTURE.md @@ -0,0 +1,79 @@ +# Project Structure + +``` +tsclient/ +ā”œā”€ā”€ src/ # Source code +ā”‚ ā”œā”€ā”€ index.ts # Main exports +ā”‚ ā””ā”€ā”€ client.ts # Custom client wrapper +ā”œā”€ā”€ tests/ # Test files +ā”‚ ā”œā”€ā”€ setup.ts # Test configuration +ā”‚ ā””ā”€ā”€ client.test.ts # Client tests +ā”œā”€ā”€ examples/ # Usage examples +ā”‚ ā””ā”€ā”€ basic-usage.ts # Basic client usage +ā”œā”€ā”€ scripts/ # Build scripts +ā”‚ ā””ā”€ā”€ build.sh # Automated build script +ā”œā”€ā”€ generated/ # Auto-generated client (from swagger) +ā”œā”€ā”€ dist/ # Compiled output +ā”œā”€ā”€ package.json # Dependencies and scripts +ā”œā”€ā”€ tsconfig.json # TypeScript configuration +ā”œā”€ā”€ jest.config.js # Jest test configuration +ā”œā”€ā”€ Makefile # Build targets +ā”œā”€ā”€ .gitignore # Git ignore rules +ā”œā”€ā”€ README.md # Main documentation +ā””ā”€ā”€ PROJECT_STRUCTURE.md # This file +``` + +## Key Components + +### 1. Generated Client (`generated/`) +- **Source**: Generated from `../http/swagger.json` +- **Tool**: OpenAPI Generator CLI +- **Language**: TypeScript with fetch API +- **Purpose**: Provides the raw API interface + +### 2. Custom Client Wrapper (`src/client.ts`) +- **Purpose**: User-friendly interface over generated client +- **Features**: + - Simplified method names + - Better error handling + - Consistent return types + - Type safety + +### 3. Main Exports (`src/index.ts`) +- **Purpose**: Clean public API +- **Exports**: + - Generated types and client + - Custom client wrapper + - Configuration interfaces + +### 4. Build System +- **Package Manager**: npm +- **Build Tool**: TypeScript compiler +- **Code Generation**: OpenAPI Generator +- **Testing**: Jest +- **Automation**: Makefile + shell scripts + +## Build Process + +1. **Generate**: `swagger.json` ā†’ TypeScript client +2. **Compile**: TypeScript ā†’ JavaScript +3. **Package**: Output to `dist/` directory + +## Development Workflow + +1. **Setup**: `make setup` or `./scripts/build.sh` +2. **Development**: Edit source files in `src/` +3. **Regenerate**: `npm run generate` (when API changes) +4. **Build**: `npm run build` or `make build` +5. **Test**: `npm test` or `make test` + +## File Purposes + +- **`package.json`**: Dependencies, scripts, metadata +- **`tsconfig.json`**: TypeScript compiler options +- **`jest.config.js`**: Test framework configuration +- **`Makefile`**: Build automation targets +- **`build.sh`**: Automated build script +- **`.gitignore`**: Version control exclusions +- **`README.md`**: User documentation +- **`PROJECT_STRUCTURE.md`**: This file (developer reference) diff --git a/market/mk20/tsclient/README.md b/market/mk20/tsclient/README.md new file mode 100644 index 000000000..b3ebc2c6f --- /dev/null +++ b/market/mk20/tsclient/README.md @@ -0,0 +1,97 @@ +# Curio TypeScript Market Client + +This is a TypeScript API client for the Curio storage market API. It provides a strongly-typed interface for interacting with Curio storage providers. + +## Installation + +```bash +npm install @curio/market-client +``` + +## Building from Source + +1. Install dependencies: +```bash +npm install +``` + +2. Generate the client from swagger files: +```bash +npm run generate +``` + +3. Compile TypeScript: +```bash +npm run compile +``` + +4. Or build everything at once: +```bash +npm run build +``` + +## Usage + +```typescript +import { MarketClient } from '@curio/market-client'; + +const client = new MarketClient({ + basePath: 'http://localhost:8080/market/mk20' +}); + +// Get supported contracts +const contracts = await client.getContracts(); + +// Get supported products +const products = await client.getProducts(); + +// Get supported data sources +const sources = await client.getSources(); + +// Get deal status +const status = await client.getStatus('deal-id-here'); + +// Submit a deal +const deal = { + // ... deal configuration +}; +const result = await client.submitDeal(deal); + +// Upload data (single request - suitable for small deals) +await client.uploadData('deal-id', [1, 2, 3, 4]); + +// Chunked upload (suitable for large deals) +await client.initializeChunkedUpload('deal-id', startUploadData); +await client.uploadChunk('deal-id', '0', chunkData); +await client.uploadChunk('deal-id', '1', chunkData); +await client.finalizeChunkedUpload('deal-id'); + +// Check upload status +const uploadStatus = await client.getUploadStatus('deal-id'); +``` + +## API Endpoints + +- `GET /contracts` - List supported DDO contracts +- `GET /products` - List supported products +- `GET /sources` - List supported data sources +- `GET /status/{id}` - Get deal status +- `POST /store` - Submit a new deal +- `PUT /upload/{id}` - Upload deal data (single request) +- `POST /upload/{id}` - Initialize chunked upload +- `PUT /uploads/{id}/{chunkNum}` - Upload a chunk +- `POST /uploads/finalize/{id}` - Finalize chunked upload +- `GET /uploads/{id}` - Get upload status + +## Development + +The client is generated from the OpenAPI/Swagger specification in `../http/swagger.json`. To regenerate after API changes: + +```bash +npm run generate +npm run compile +``` + +## License + +MIT diff --git a/market/mk20/tsclient/examples/basic-usage.ts b/market/mk20/tsclient/examples/basic-usage.ts new file mode 100644 index 000000000..84141bb1e --- /dev/null +++ b/market/mk20/tsclient/examples/basic-usage.ts @@ -0,0 +1,86 @@ +import { Client, MarketClientConfig, Deal, DataSource, Products, DDOV1 } from '../src'; + +// Example configuration +const config: MarketClientConfig = { + basePath: 'http://localhost:8080/market/mk20', + // Optional: Add custom headers + headers: { + 'Authorization': 'Bearer your-token-here' + } +}; + +// Create client instance +const client = new Client(config); + +async function exampleUsage() { + try { + // Get supported contracts + console.log('Getting supported contracts...'); + const contracts = await client.getContracts(); + console.log('Contracts:', contracts); + + // Get supported products + console.log('\nGetting supported products...'); + const products = await client.getProducts(); + console.log('Products:', products); + + // Get supported data sources + console.log('\nGetting supported data sources...'); + const sources = await client.getSources(); + console.log('Sources:', sources); + + // Example: Submit a deal + console.log('\nSubmitting a deal...'); + const deal: Deal = { + identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], // Example identifier + client: 'f1abcdefghijklmnopqrstuvwxyz123456789', + data: { + piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + format: { + raw: {} + }, + source_httpput: { + raw_size: 1024 * 1024 // 1MB + } + } as DataSource, + products: { + ddo_v1: { + duration: 518400, // Minimum duration in epochs + provider: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, + contractAddress: '0x1234567890123456789012345678901234567890', + contractVerifyMethod: 'verifyDeal', + contractVerifyMethodParams: [], + pieceManager: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, + notificationAddress: 'f1abcdefghijklmnopqrstuvwxyz123456789', + notificationPayload: [] + } as DDOV1 + } as Products + }; + + const result = await client.submitDeal(deal); + console.log('Deal submitted:', result); + + // Get deal status + if (result && result === 200) { // DealCode.Ok + console.log('\nGetting deal status...'); + const status = await client.getStatus('example-deal-id'); + console.log('Deal status:', status); + } + + } catch (error) { + console.error('Error:', error); + } +} + +// Example: Upload data for a deal +async function uploadDataExample(dealId: string, data: number[]) { + try { + console.log(`Uploading data for deal ${dealId}...`); + await client.uploadData(dealId, data); + console.log('Data uploaded successfully'); + } catch (error) { + console.error('Upload failed:', error); + } +} + +export { exampleUsage, uploadDataExample }; diff --git a/market/mk20/tsclient/examples/upload-methods.ts b/market/mk20/tsclient/examples/upload-methods.ts new file mode 100644 index 000000000..d83037a71 --- /dev/null +++ b/market/mk20/tsclient/examples/upload-methods.ts @@ -0,0 +1,229 @@ +import { Client, MarketClientConfig, StartUpload } from '../src'; + +// Example configuration +const config: MarketClientConfig = { + basePath: 'http://localhost:8080/market/mk20', + headers: { + 'Authorization': 'Bearer your-token-here' + } +}; + +// Create client instance +const client = new Client(config); + +// Example 1: Single upload (suitable for small deals) +async function singleUploadExample(dealId: string, data: number[]) { + try { + console.log(`Uploading ${data.length} bytes for deal ${dealId}...`); + await client.uploadData(dealId, data); + console.log('Single upload completed successfully'); + } catch (error) { + console.error('Single upload failed:', error); + } +} + +// Example 2: Chunked upload (suitable for large deals) +async function chunkedUploadExample(dealId: string, largeData: number[], chunkSize: number = 1024 * 1024) { + try { + console.log(`Starting chunked upload for deal ${dealId}...`); + + // Step 1: Initialize the upload + const startUpload: StartUpload = { + rawSize: largeData.length, + chunkSize: chunkSize + }; + + const initResult = await client.initializeChunkedUpload(dealId, startUpload); + console.log('Upload initialized with result:', initResult); + + // Step 2: Upload data in chunks + const chunks: Array<{ chunkNum: string; result: number }> = []; + for (let i = 0; i < largeData.length; i += chunkSize) { + const chunk = largeData.slice(i, i + chunkSize); + const chunkNum = Math.floor(i / chunkSize).toString(); + + console.log(`Uploading chunk ${chunkNum} (${chunk.length} bytes)...`); + const uploadResult = await client.uploadChunk(dealId, chunkNum, chunk); + chunks.push({ chunkNum, result: uploadResult }); + + // Optional: Check upload status periodically + if (chunks.length % 10 === 0) { + const status = await client.getUploadStatus(dealId); + console.log(`Upload status after ${chunks.length} chunks:`, status); + } + } + + console.log(`All ${chunks.length} chunks uploaded successfully`); + + // Step 3: Finalize the upload + console.log('Finalizing upload...'); + const finalizeResult = await client.finalizeChunkedUpload(dealId); + console.log('Upload finalized with result:', finalizeResult); + + console.log('Chunked upload completed successfully'); + + } catch (error) { + console.error('Chunked upload failed:', error); + } +} + +// Example 3: Parallel chunk uploads for better performance +async function parallelChunkUploadExample(dealId: string, largeData: number[], chunkSize: number = 1024 * 1024) { + try { + console.log(`Starting parallel chunked upload for deal ${dealId}...`); + + // Step 1: Initialize the upload + const startUpload: StartUpload = { + rawSize: largeData.length, + chunkSize: chunkSize + }; + + await client.initializeChunkedUpload(dealId, startUpload); + console.log('Upload initialized'); + + // Step 2: Prepare all chunks + const chunks: Array<{ chunkNum: string; data: number[] }> = []; + for (let i = 0; i < largeData.length; i += chunkSize) { + const chunk = largeData.slice(i, i + chunkSize); + const chunkNum = Math.floor(i / chunkSize).toString(); + chunks.push({ chunkNum, data: chunk }); + } + + console.log(`Uploading ${chunks.length} chunks in parallel...`); + + // Step 3: Upload chunks in parallel (with concurrency limit) + const concurrencyLimit = 5; // Limit concurrent requests + const results: Array<{ chunkNum: string; result: number }> = []; + + for (let i = 0; i < chunks.length; i += concurrencyLimit) { + const batch = chunks.slice(i, i + concurrencyLimit); + const batchPromises = batch.map(async ({ chunkNum, data }) => { + const result = await client.uploadChunk(dealId, chunkNum, data); + return { chunkNum, result }; + }); + + const batchResults = await Promise.all(batchPromises); + results.push(...batchResults); + + console.log(`Completed batch ${Math.floor(i / concurrencyLimit) + 1}/${Math.ceil(chunks.length / concurrencyLimit)}`); + } + + console.log(`All ${results.length} chunks uploaded successfully`); + + // Step 4: Finalize the upload + console.log('Finalizing upload...'); + const finalizeResult = await client.finalizeChunkedUpload(dealId); + console.log('Upload finalized with result:', finalizeResult); + + console.log('Parallel chunked upload completed successfully'); + + } catch (error) { + console.error('Parallel chunked upload failed:', error); + } +} + +// Example 4: Monitor upload progress +async function monitoredUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024) { + try { + console.log(`Starting monitored upload for deal ${dealId}...`); + + // Initialize upload + const startUpload: StartUpload = { + rawSize: data.length, + chunkSize: chunkSize + }; + + await client.initializeChunkedUpload(dealId, startUpload); + + // Upload with progress monitoring + const totalChunks = Math.ceil(data.length / chunkSize); + let completedChunks = 0; + + for (let i = 0; i < data.length; i += chunkSize) { + const chunk = data.slice(i, i + chunkSize); + const chunkNum = Math.floor(i / chunkSize).toString(); + + await client.uploadChunk(dealId, chunkNum, chunk); + completedChunks++; + + // Show progress + const progress = ((completedChunks / totalChunks) * 100).toFixed(1); + console.log(`Progress: ${progress}% (${completedChunks}/${totalChunks} chunks)`); + + // Check status every 10 chunks + if (completedChunks % 10 === 0) { + const status = await client.getUploadStatus(dealId); + console.log('Current upload status:', status); + } + } + + // Finalize + const finalizeResult = await client.finalizeChunkedUpload(dealId); + console.log('Upload completed and finalized:', finalizeResult); + + } catch (error) { + console.error('Monitored upload failed:', error); + } +} + +// Example 5: Error handling and retry logic +async function robustUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024, maxRetries: number = 3) { + try { + console.log(`Starting robust upload for deal ${dealId}...`); + + // Initialize upload + const startUpload: StartUpload = { + rawSize: data.length, + chunkSize: chunkSize + }; + + await client.initializeChunkedUpload(dealId, startUpload); + + // Upload with retry logic + const totalChunks = Math.ceil(data.length / chunkSize); + let completedChunks = 0; + + for (let i = 0; i < data.length; i += chunkSize) { + const chunk = data.slice(i, i + chunkSize); + const chunkNum = Math.floor(i / chunkSize).toString(); + + let retries = 0; + let success = false; + + while (!success && retries < maxRetries) { + try { + await client.uploadChunk(dealId, chunkNum, chunk); + success = true; + completedChunks++; + console.log(`Chunk ${chunkNum} uploaded successfully (${completedChunks}/${totalChunks})`); + } catch (error) { + retries++; + console.warn(`Chunk ${chunkNum} upload failed (attempt ${retries}/${maxRetries}):`, error); + + if (retries >= maxRetries) { + throw new Error(`Failed to upload chunk ${chunkNum} after ${maxRetries} attempts`); + } + + // Wait before retry (exponential backoff) + await new Promise(resolve => setTimeout(resolve, Math.pow(2, retries) * 1000)); + } + } + } + + // Finalize + const finalizeResult = await client.finalizeChunkedUpload(dealId); + console.log('Robust upload completed successfully:', finalizeResult); + + } catch (error) { + console.error('Robust upload failed:', error); + throw error; + } +} + +export { + singleUploadExample, + chunkedUploadExample, + parallelChunkUploadExample, + monitoredUploadExample, + robustUploadExample +}; diff --git a/market/mk20/tsclient/jest.config.js b/market/mk20/tsclient/jest.config.js new file mode 100644 index 000000000..569828795 --- /dev/null +++ b/market/mk20/tsclient/jest.config.js @@ -0,0 +1,17 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/src', '/tests'], + testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'], + transform: { + '^.+\\.ts$': 'ts-jest', + }, + collectCoverageFrom: [ + 'src/**/*.ts', + '!src/**/*.d.ts', + ], + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov', 'html'], + moduleFileExtensions: ['ts', 'js', 'json'], + setupFilesAfterEnv: ['/tests/setup.ts'], +}; diff --git a/market/mk20/tsclient/openapitools.json b/market/mk20/tsclient/openapitools.json new file mode 100644 index 000000000..a82623d64 --- /dev/null +++ b/market/mk20/tsclient/openapitools.json @@ -0,0 +1,7 @@ +{ + "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", + "spaces": 2, + "generator-cli": { + "version": "7.14.0" + } +} diff --git a/market/mk20/tsclient/package-lock.json b/market/mk20/tsclient/package-lock.json new file mode 100644 index 000000000..a24ee07f2 --- /dev/null +++ b/market/mk20/tsclient/package-lock.json @@ -0,0 +1,5837 @@ +{ + "name": "@curio/market-client", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@curio/market-client", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "isomorphic-fetch": "^3.0.0" + }, + "devDependencies": { + "@openapitools/openapi-generator-cli": "^2.7.0", + "@types/jest": "^29.0.0", + "@types/node": "^20.0.0", + "jest": "^29.0.0", + "ts-jest": "^29.0.0", + "typescript": "^5.0.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", + "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.3.tgz", + "integrity": "sha512-yDBHV9kQNcr2/sUr9jghVyz9C3Y5G2zUM2H2lo+9mKv4sFgbA8s8Z9t8D1jiTkGoO/NoIfKMyKWr4s6CN23ZwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.3", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helpers": "^7.28.3", + "@babel/parser": "^7.28.3", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.3", + "@babel/types": "^7.28.2", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.3.tgz", + "integrity": "sha512-3lSpxGgvnmZznmBkCRnVREPUFJv2wrv9iAoFDvADJc0ypmdOxdUtcLeBgBJ6zE0PMeTKnxeQzyk0xTBq4Ep7zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.3", + "@babel/types": "^7.28.2", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.3.tgz", + "integrity": "sha512-PTNtvUQihsAsDHMOP5pfobP8C6CM4JWXmP8DrEIt46c3r2bf87Ua1zoqevsMo9g+tWDwgWrFP5EIxuBx5RudAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.3.tgz", + "integrity": "sha512-7+Ey1mAgYqFAx2h0RuoxcQT5+MlG3GTV0TQrgr7/ZliKsm/MNDxVVutlWaziMq7wJNAz8MTqz55XLpWvva6StA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", + "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.3.tgz", + "integrity": "sha512-7w4kZYHneL3A6NP2nxzHvT3HCZ7puDZZjFMqDpBPECub79sTtSO5CGXDkKrTQq8ksAwfD/XI2MRFX23njdDaIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.3", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.3", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.2", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.2", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.2.tgz", + "integrity": "sha512-ruv7Ae4J5dUYULmeXw1gmb7rYRz57OWCPM57pHojnLq/3Z1CK2lNSLTCVjxVk1F/TZHwOZZrOWi0ur95BbLxNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@borewit/text-codec": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.1.1.tgz", + "integrity": "sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.1.tgz", + "integrity": "sha512-Oau4yL24d2B5IL4ma4UpbQigkVhzPDXLoqy1ggK4gnHg/stmkffJE4oOXHXF3uz0UEpywG68KcyXsyYpA1Re/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "chardet": "^2.1.0", + "iconv-lite": "^0.6.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.0.tgz", + "integrity": "sha512-TKY5pyBkHyADOPYlRT9Lx6F544mPl0vS5Ew7BJ45hA08Q+t3GjbueLliBWN3sMICk6+y7HdyxSzC4bWS8baBdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/reporters/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@jest/reporters/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.30", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", + "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@nestjs/axios": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-4.0.1.tgz", + "integrity": "sha512-68pFJgu+/AZbWkGu65Z3r55bTsCPlgyKaV4BSG8yUAD72q1PPuyVRgUwFv6BxdnibTUHlyxm06FmYWNC+bjN7A==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "axios": "^1.3.1", + "rxjs": "^7.0.0" + } + }, + "node_modules/@nestjs/common": { + "version": "11.1.6", + "resolved": "https://registry.npmjs.org/@nestjs/common/-/common-11.1.6.tgz", + "integrity": "sha512-krKwLLcFmeuKDqngG2N/RuZHCs2ycsKcxWIDgcm7i1lf3sQ0iG03ci+DsP/r3FcT/eJDFsIHnKtNta2LIi7PzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "file-type": "21.0.0", + "iterare": "1.2.1", + "load-esm": "1.0.2", + "tslib": "2.8.1", + "uid": "2.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nest" + }, + "peerDependencies": { + "class-transformer": ">=0.4.1", + "class-validator": ">=0.13.2", + "reflect-metadata": "^0.1.12 || ^0.2.0", + "rxjs": "^7.1.0" + }, + "peerDependenciesMeta": { + "class-transformer": { + "optional": true + }, + "class-validator": { + "optional": true + } + } + }, + "node_modules/@nestjs/core": { + "version": "11.1.6", + "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-11.1.6.tgz", + "integrity": "sha512-siWX7UDgErisW18VTeJA+x+/tpNZrJewjTBsRPF3JVxuWRuAB1kRoiJcxHgln8Lb5UY9NdvklITR84DUEXD0Cg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@nuxt/opencollective": "0.4.1", + "fast-safe-stringify": "2.1.1", + "iterare": "1.2.1", + "path-to-regexp": "8.2.0", + "tslib": "2.8.1", + "uid": "2.0.2" + }, + "engines": { + "node": ">= 20" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nest" + }, + "peerDependencies": { + "@nestjs/common": "^11.0.0", + "@nestjs/microservices": "^11.0.0", + "@nestjs/platform-express": "^11.0.0", + "@nestjs/websockets": "^11.0.0", + "reflect-metadata": "^0.1.12 || ^0.2.0", + "rxjs": "^7.1.0" + }, + "peerDependenciesMeta": { + "@nestjs/microservices": { + "optional": true + }, + "@nestjs/platform-express": { + "optional": true + }, + "@nestjs/websockets": { + "optional": true + } + } + }, + "node_modules/@nuxt/opencollective": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@nuxt/opencollective/-/opencollective-0.4.1.tgz", + "integrity": "sha512-GXD3wy50qYbxCJ652bDrDzgMr3NFEkIS374+IgFQKkCvk9yiYcLvX2XDYr7UyQxf4wK0e+yqDYRubZ0DtOxnmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "consola": "^3.2.3" + }, + "bin": { + "opencollective": "bin/opencollective.js" + }, + "engines": { + "node": "^14.18.0 || >=16.10.0", + "npm": ">=5.10.0" + } + }, + "node_modules/@nuxtjs/opencollective": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@nuxtjs/opencollective/-/opencollective-0.3.2.tgz", + "integrity": "sha512-um0xL3fO7Mf4fDxcqx9KryrB7zgRM5JSlvGN5AGkP6JLM5XEKyjeAiPbNxdXVXQ16isuAhYpvP88NgL2BGd6aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "consola": "^2.15.0", + "node-fetch": "^2.6.1" + }, + "bin": { + "opencollective": "bin/opencollective.js" + }, + "engines": { + "node": ">=8.0.0", + "npm": ">=5.0.0" + } + }, + "node_modules/@nuxtjs/opencollective/node_modules/consola": { + "version": "2.15.3", + "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz", + "integrity": "sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@openapitools/openapi-generator-cli": { + "version": "2.23.1", + "resolved": "https://registry.npmjs.org/@openapitools/openapi-generator-cli/-/openapi-generator-cli-2.23.1.tgz", + "integrity": "sha512-Kd5EZqzbcIXf6KRlpUrheHMzQNRHsJWzAGrm4ncWCNhnQl+Mh6TsFcqq+hIetgiFCknWBH6cZ2f37SxPxaon4w==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@nestjs/axios": "4.0.1", + "@nestjs/common": "11.1.6", + "@nestjs/core": "11.1.6", + "@nuxtjs/opencollective": "0.3.2", + "axios": "1.11.0", + "chalk": "4.1.2", + "commander": "8.3.0", + "compare-versions": "4.1.4", + "concurrently": "9.2.1", + "console.table": "0.10.0", + "fs-extra": "11.3.1", + "glob": "11.0.3", + "inquirer": "8.2.7", + "proxy-agent": "6.5.0", + "reflect-metadata": "0.2.2", + "rxjs": "7.8.2", + "tslib": "2.8.1" + }, + "bin": { + "openapi-generator-cli": "main.js" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/openapi_generator" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@tokenizer/inflate": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", + "integrity": "sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "fflate": "^0.8.2", + "token-types": "^6.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "node_modules/@types/node": { + "version": "20.19.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.11.tgz", + "integrity": "sha512-uug3FEEGv0r+jrecvUUpbY8lLisvIjg6AAic6a2bSP5OEOLeJsDSnvhCDov7ipFFMXS3orMpzlmi0ZcuGkBbow==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", + "integrity": "sha512-1Lx3WLFQWm3ooKDYZD1eXmoGO9fxYQjrycfHFC8P0sCfQVXyROp0p9PFWBehewBOdCwHc+f/b8I0fMto5eSfwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", + "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.25.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.4.tgz", + "integrity": "sha512-4jYpcjabC606xJ3kw2QwGEZKX0Aw7sgQdZCvIK9dhVSPh76BKo+C+btT1RRofH7B+8iNpEbgGNVWiLki5q93yg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001737", + "electron-to-chromium": "^1.5.211", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bs-logger": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", + "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-json-stable-stringify": "2.x" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001739", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001739.tgz", + "integrity": "sha512-y+j60d6ulelrNSwpPyrHdl+9mJnQzHBr08xm48Qno0nSk4h3Qojh+ziv2qE6rXf4k3tadF4o1J/1tAbVm1NtnA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/chardet": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.0.tgz", + "integrity": "sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", + "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/compare-versions": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-4.1.4.tgz", + "integrity": "sha512-FemMreK9xNyL8gQevsdRMrvO4lFCkQP7qbuktn1q8ndcNk1+0mz7lgE7b/sNvbhVgY4w6tMN1FDp6aADjqw2rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/concurrently": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.1.tgz", + "integrity": "sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "4.1.2", + "rxjs": "7.8.2", + "shell-quote": "1.8.3", + "supports-color": "8.1.1", + "tree-kill": "1.2.2", + "yargs": "17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/console.table": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/console.table/-/console.table-0.10.0.tgz", + "integrity": "sha512-dPyZofqggxuvSf7WXvNjuRfnsOk1YazkVP8FdxH4tcH2c37wc79/Yl6Bhr7Lsu00KMgy2ql/qCMuNu8xctZM8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "easy-table": "1.1.0" + }, + "engines": { + "node": "> 0.10" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dedent": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.6.0.tgz", + "integrity": "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/easy-table": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/easy-table/-/easy-table-1.1.0.tgz", + "integrity": "sha512-oq33hWOSSnl2Hoh00tZWaIPi1ievrD9aFG82/IgjlycAnW9hHx5PkJiXpxPsgEE+H7BsbVQXFVFST8TEXS6/pA==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "wcwidth": ">=1.0.1" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.211", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.211.tgz", + "integrity": "sha512-IGBvimJkotaLzFnwIVgW9/UD/AOJ2tByUmeOrtqBfACSbAw5b1G0XpvdaieKyc7ULmbwXVx+4e4Be8pOPBrYkw==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "dev": true, + "license": "MIT" + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/file-type": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.0.0.tgz", + "integrity": "sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.2.7", + "strtok3": "^10.2.2", + "token-types": "^6.0.0", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fs-extra": { + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.1.tgz", + "integrity": "sha512-eXvGGwZ5CL17ZSwHWd3bbgk7UUpF6IFHtP57NYYakPvHOs8GDgDe5KJI36jIJzDkJ6eJjuzRA8eBQb6SkKue0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/glob": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", + "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.0.3", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "8.2.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", + "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.0", + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.1", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "figures": "^3.0.0", + "lodash": "^4.17.21", + "mute-stream": "0.0.8", + "ora": "^5.4.1", + "run-async": "^2.4.0", + "rxjs": "^7.5.5", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6", + "wrap-ansi": "^6.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "license": "MIT", + "dependencies": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/iterare": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iterare/-/iterare-1.2.1.tgz", + "integrity": "sha512-RKYVTCjAnRthyJes037NX/IiqeidgN1xc3j1RjFfECFp28A1GVwK9nA+i0rJPaHqSZwygLzRnFlzUuHFoWWy+Q==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=6" + } + }, + "node_modules/jackspeak": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-config/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-config/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jest-runtime/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/load-esm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/load-esm/-/load-esm-1.0.2.tgz", + "integrity": "sha512-nVAvWk/jeyrWyXEAs84mpQCYccxRqgKY4OznLuJhJCa0XsPSfdOIr2zvBZEj3IHEHbX97jjscKRRV539bW0Gpw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + }, + { + "type": "buymeacoffee", + "url": "https://buymeacoffee.com/borewit" + } + ], + "license": "MIT", + "engines": { + "node": ">=13.2.0" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.3.tgz", + "integrity": "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==", + "dev": true, + "license": "ISC", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true, + "license": "ISC" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dev": true, + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", + "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", + "dev": true, + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/path-to-regexp": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true, + "license": "MIT" + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/restore-cursor/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", + "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strtok3": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.4.tgz", + "integrity": "sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/token-types": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.1.tgz", + "integrity": "sha512-kh9LVIWH5CnL63Ipf0jhlBIy0UsrMj/NJDfpsy1SqOXlLKEVyXXYrnFxFT1yOOYVGBSApeVnjPw/sBz5BfEjAQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.1.0", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "license": "MIT", + "bin": { + "tree-kill": "cli.js" + } + }, + "node_modules/ts-jest": { + "version": "29.4.1", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.1.tgz", + "integrity": "sha512-SaeUtjfpg9Uqu8IbeDKtdaS0g8lS6FT6OzM3ezrDfErPJPHNDo/Ey+VFGP1bQIDfagYDLyRpd7O15XpG1Es2Uw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bs-logger": "^0.2.6", + "fast-json-stable-stringify": "^2.1.0", + "handlebars": "^4.7.8", + "json5": "^2.2.3", + "lodash.memoize": "^4.1.2", + "make-error": "^1.3.6", + "semver": "^7.7.2", + "type-fest": "^4.41.0", + "yargs-parser": "^21.1.1" + }, + "bin": { + "ts-jest": "cli.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": ">=7.0.0-beta.0 <8", + "@jest/transform": "^29.0.0 || ^30.0.0", + "@jest/types": "^29.0.0 || ^30.0.0", + "babel-jest": "^29.0.0 || ^30.0.0", + "jest": "^29.0.0 || ^30.0.0", + "jest-util": "^29.0.0 || ^30.0.0", + "typescript": ">=4.3 <6" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "@jest/transform": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "babel-jest": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jest-util": { + "optional": true + } + } + }, + "node_modules/ts-jest/node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ts-jest/node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", + "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/uid": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", + "integrity": "sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "dev": true, + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", + "license": "MIT" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/write-file-atomic/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json new file mode 100644 index 000000000..3fc296331 --- /dev/null +++ b/market/mk20/tsclient/package.json @@ -0,0 +1,41 @@ +{ + "name": "@curio/market-client", + "version": "1.0.0", + "description": "TypeScript API client for Curio storage market", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "npm run generate && npm run compile", + "generate": "openapi-generator-cli generate -i ../http/swagger.json -g typescript-fetch -o ./generated --additional-properties=supportsES6=true,typescriptThreePlus=true", + "compile": "tsc", + "clean": "rm -rf dist generated", + "test": "jest", + "test:watch": "jest --watch", + "test:coverage": "jest --coverage", + "prepublishOnly": "npm run build" + }, + "keywords": [ + "curio", + "filecoin", + "storage", + "market", + "api", + "typescript" + ], + "author": "Curio Team", + "license": "MIT", + "devDependencies": { + "@openapitools/openapi-generator-cli": "^2.7.0", + "@types/node": "^20.0.0", + "@types/jest": "^29.0.0", + "jest": "^29.0.0", + "ts-jest": "^29.0.0", + "typescript": "^5.0.0" + }, + "dependencies": { + "isomorphic-fetch": "^3.0.0" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/market/mk20/tsclient/scripts/build.sh b/market/mk20/tsclient/scripts/build.sh new file mode 100755 index 000000000..fc6f06b2b --- /dev/null +++ b/market/mk20/tsclient/scripts/build.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +echo "šŸš€ Building Curio TypeScript Market Client..." + +# Check if Node.js is installed +if ! command -v node &> /dev/null; then + echo "āŒ Node.js is not installed. Please install Node.js 18+ first." + exit 1 +fi + +# Check Node.js version +NODE_VERSION=$(node -v | cut -d'v' -f2 | cut -d'.' -f1) +if [ "$NODE_VERSION" -lt 18 ]; then + echo "āŒ Node.js version 18+ is required. Current version: $(node -v)" + exit 1 +fi + +echo "āœ… Node.js version: $(node -v)" + +# Check if npm is installed +if ! command -v npm &> /dev/null; then + echo "āŒ npm is not installed. Please install npm first." + exit 1 +fi + +echo "āœ… npm version: $(npm -v)" + +# Clean previous builds +echo "šŸ§¹ Cleaning previous builds..." +npm run clean + +# Install dependencies +echo "šŸ“¦ Installing dependencies..." +npm install + +# Generate client from swagger +echo "šŸ”§ Generating TypeScript client from swagger files..." +npm run generate + +# Compile TypeScript +echo "āš™ļø Compiling TypeScript..." +npm run compile + +echo "āœ… Build completed successfully!" +echo "" +echo "šŸ“ Generated files:" +echo " - Generated client: ./generated/" +echo " - Compiled output: ./dist/" +echo " - Type definitions: ./dist/index.d.ts" +echo "" +echo "šŸš€ You can now use the client:" +echo " import { Client } from '@curio/market-client';" +echo "" +echo "šŸ“š See examples/ for usage examples" diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts new file mode 100644 index 000000000..2aff2f616 --- /dev/null +++ b/market/mk20/tsclient/src/client.ts @@ -0,0 +1,140 @@ +import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources } from '../generated'; +import { Configuration } from '../generated/runtime'; + +export interface MarketClientConfig extends ConfigurationParameters { + basePath: string; +} + +export class MarketClient { + private api: DefaultApi; + + constructor(config: MarketClientConfig) { + this.api = new DefaultApi(new Configuration(config)); + } + + /** + * Get supported DDO contracts + */ + async getContracts(): Promise { + try { + const response = await this.api.contractsGet(); + return response.contracts || []; + } catch (error) { + throw new Error(`Failed to get contracts: ${error}`); + } + } + + /** + * Get supported products + */ + async getProducts(): Promise { + try { + const response = await this.api.productsGet(); + return response; + } catch (error) { + throw new Error(`Failed to get products: ${error}`); + } + } + + /** + * Get supported data sources + */ + async getSources(): Promise { + try { + const response = await this.api.sourcesGet(); + return response; + } catch (error) { + throw new Error(`Failed to get sources: ${error}`); + } + } + + /** + * Get deal status by ID + */ + async getStatus(id: string): Promise { + try { + const response = await this.api.statusIdGet({ id }); + return response; + } catch (error) { + throw new Error(`Failed to get deal status for ${id}: ${error}`); + } + } + + /** + * Submit a new deal + */ + async submitDeal(deal: Mk20Deal): Promise { + try { + const response = await this.api.storePost({ body: deal }); + return response; + } catch (error) { + throw new Error(`Failed to submit deal: ${error}`); + } + } + + /** + * Upload deal data + */ + async uploadData(id: string, data: Array): Promise { + try { + await this.api.uploadIdPut({ id, body: data }); + } catch (error) { + throw new Error(`Failed to upload data for deal ${id}: ${error}`); + } + } + + /** + * Initialize chunked upload for a deal + * @param id - Deal identifier + * @param startUpload - Upload initialization data + */ + async initializeChunkedUpload(id: string, startUpload: any): Promise { + try { + const result = await this.api.uploadsIdPost({ id, data: startUpload }); + return result; + } catch (error) { + throw new Error(`Failed to initialize chunked upload for deal ${id}: ${error}`); + } + } + + /** + * Upload a chunk of data for a deal + * @param id - Deal identifier + * @param chunkNum - Chunk number + * @param data - Chunk data + */ + async uploadChunk(id: string, chunkNum: string, data: Array): Promise { + try { + const result = await this.api.uploadsIdChunkNumPut({ id, chunkNum, data }); + return result; + } catch (error) { + throw new Error(`Failed to upload chunk ${chunkNum} for deal ${id}: ${error}`); + } + } + + /** + * Finalize chunked upload for a deal + * @param id - Deal identifier + * @param deal - Optional deal data for finalization + */ + async finalizeChunkedUpload(id: string, deal?: any): Promise { + try { + const result = await this.api.uploadsFinalizeIdPost({ id, body: deal }); + return result; + } catch (error) { + throw new Error(`Failed to finalize chunked upload for deal ${id}: ${error}`); + } + } + + /** + * Get upload status for a deal + * @param id - Deal identifier + */ + async getUploadStatus(id: string): Promise { + try { + return await this.api.uploadsIdGet({ id }); + } catch (error) { + throw new Error(`Failed to get upload status for deal ${id}: ${error}`); + } + } +} diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts new file mode 100644 index 000000000..be46b75da --- /dev/null +++ b/market/mk20/tsclient/src/index.ts @@ -0,0 +1,36 @@ +// Export the generated client and types +export * from '../generated'; + +// Re-export commonly used types for convenience +export type { + Mk20Deal as Deal, + Mk20DataSource as DataSource, + Mk20Products as Products, + Mk20DDOV1 as DDOV1, + Mk20PDPV1 as PDPV1, + Mk20RetrievalV1 as RetrievalV1, + Mk20DealProductStatusResponse as DealProductStatusResponse, + Mk20SupportedContracts as SupportedContracts, + Mk20SupportedProducts as SupportedProducts, + Mk20SupportedDataSources as SupportedDataSources, + Mk20DealCode as DealCode +} from '../generated'; + +// Export the main client class +export { DefaultApi as MarketClient } from '../generated'; + +// Export the custom client wrapper +export { MarketClient as Client } from './client'; +export type { MarketClientConfig } from './client'; + +// Re-export configuration types +export type { Configuration } from '../generated'; + +// Re-export upload-related types for convenience +export type { + Mk20StartUpload as StartUpload, + Mk20UploadCode as UploadCode, + Mk20UploadStartCode as UploadStartCode, + Mk20UploadStatus as UploadStatus, + Mk20UploadStatusCode as UploadStatusCode +} from '../generated'; diff --git a/market/mk20/tsclient/tests/client.test.ts b/market/mk20/tsclient/tests/client.test.ts new file mode 100644 index 000000000..59a263ac0 --- /dev/null +++ b/market/mk20/tsclient/tests/client.test.ts @@ -0,0 +1,121 @@ +import { MarketClient, MarketClientConfig } from '../src/client'; +import { mockResponse, mockError } from './setup'; + +// Mock the generated API +jest.mock('../generated', () => ({ + DefaultApi: jest.fn().mockImplementation(() => ({ + contractsGet: jest.fn(), + productsGet: jest.fn(), + sourcesGet: jest.fn(), + statusIdGet: jest.fn(), + storePost: jest.fn(), + uploadIdPut: jest.fn(), + })), +})); + +describe('MarketClient', () => { + let client: MarketClient; + let mockApi: any; + + beforeEach(() => { + const config: MarketClientConfig = { + basePath: 'http://localhost:8080/market/mk20', + } as MarketClientConfig; + + client = new MarketClient(config); + + // Get the mocked API instance and set up the mocks + const { DefaultApi } = require('../generated'); + mockApi = new DefaultApi(); + + // Set up the mock methods on the client's API instance + (client as any).api = mockApi; + }); + + describe('getContracts', () => { + it('should return contracts successfully', async () => { + const mockContracts = ['0x123', '0x456']; + mockApi.contractsGet.mockResolvedValue({ + contracts: mockContracts, + }); + + const result = await client.getContracts(); + expect(result).toEqual(mockContracts); + }); + + it('should handle errors gracefully', async () => { + mockApi.contractsGet.mockRejectedValue(new Error('API Error')); + + await expect(client.getContracts()).rejects.toThrow('Failed to get contracts: Error: API Error'); + }); + }); + + describe('getProducts', () => { + it('should return products successfully', async () => { + const mockProducts = { ddo_v1: true, pdp_v1: true }; + mockApi.productsGet.mockResolvedValue(mockProducts); + + const result = await client.getProducts(); + expect(result).toEqual(mockProducts); + }); + }); + + describe('getSources', () => { + it('should return sources successfully', async () => { + const mockSources = { http: true, aggregate: true }; + mockApi.sourcesGet.mockResolvedValue(mockSources); + + const result = await client.getSources(); + expect(result).toEqual(mockSources); + }); + }); + + describe('getStatus', () => { + it('should return deal status successfully', async () => { + const mockStatus = { identifier: 'test-id', status: 'active' }; + mockApi.statusIdGet.mockResolvedValue(mockStatus); + + const result = await client.getStatus('test-id'); + expect(result).toEqual(mockStatus); + }); + + it('should handle errors with deal ID context', async () => { + mockApi.statusIdGet.mockRejectedValue(new Error('Not Found')); + + await expect(client.getStatus('test-id')).rejects.toThrow('Failed to get deal status for test-id: Error: Not Found'); + }); + }); + + describe('submitDeal', () => { + it('should submit deal successfully', async () => { + const mockDeal = { identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] }; + const mockResult = 200; // DealCode.Ok + mockApi.storePost.mockResolvedValue(mockResult); + + const result = await client.submitDeal(mockDeal); + expect(result).toEqual(mockResult); + }); + }); + + describe('uploadData', () => { + it('should upload data successfully', async () => { + const testData = [1, 2, 3, 4, 5, 6, 7, 8]; + mockApi.uploadIdPut.mockResolvedValue(undefined); + + await expect(client.uploadData('test-id', testData)).resolves.not.toThrow(); + }); + + it('should handle upload errors', async () => { + const testData = [1, 2, 3, 4, 5, 6, 7, 8]; + mockApi.uploadIdPut.mockRejectedValue(new Error('Upload failed')); + + await expect(client.uploadData('test-id', testData)).rejects.toThrow('Failed to upload data for deal test-id: Error: Upload failed'); + }); + }); + + describe('getInfo', () => { + it('should handle info endpoint not available', async () => { + await expect(client.getInfo()).rejects.toThrow('Failed to get info: Error: Info endpoint not available in generated API'); + }); + }); +}); diff --git a/market/mk20/tsclient/tests/setup.ts b/market/mk20/tsclient/tests/setup.ts new file mode 100644 index 000000000..b35a083e6 --- /dev/null +++ b/market/mk20/tsclient/tests/setup.ts @@ -0,0 +1,24 @@ +// Global test setup +import 'isomorphic-fetch'; + +// Mock fetch for testing +global.fetch = jest.fn(); + +// Reset mocks before each test +beforeEach(() => { + jest.clearAllMocks(); +}); + +// Test utilities +export const mockResponse = (data: any, status = 200) => { + return Promise.resolve({ + ok: status >= 200 && status < 300, + status, + json: () => Promise.resolve(data), + text: () => Promise.resolve(JSON.stringify(data)), + } as Response); +}; + +export const mockError = (status: number, message: string) => { + return Promise.reject(new Error(`${status}: ${message}`)); +}; diff --git a/market/mk20/tsclient/tsconfig.json b/market/mk20/tsclient/tsconfig.json new file mode 100644 index 000000000..8828dcc83 --- /dev/null +++ b/market/mk20/tsclient/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "declaration": true, + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true + }, + "include": [ + "src/**/*", + "generated/**/*" + ], + "exclude": [ + "node_modules", + "dist" + ] +} From 73bc33a178689b2cc778612fb23d57ed34202c44 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Tue, 2 Sep 2025 18:43:19 -0500 Subject: [PATCH 29/55] CidV2ts, ez-upload --- lib/commcidv2/commcidv2_test.go | 409 ++++++++++++++++++ market/mk20/tsclient/README.md | 27 +- .../mk20/tsclient/examples/pdpv1-workflow.ts | 87 ++++ .../examples/piece-cid-computation.ts | 133 ++++++ market/mk20/tsclient/jest.config.js | 3 + market/mk20/tsclient/package-lock.json | 30 +- market/mk20/tsclient/package.json | 7 +- market/mk20/tsclient/src/client.ts | 339 ++++++++++++++- market/mk20/tsclient/src/index.ts | 3 + .../tests/__mocks__/multiformats/cid.ts | 17 + .../__mocks__/multiformats/hashes/sha2.ts | 8 + 11 files changed, 1058 insertions(+), 5 deletions(-) create mode 100644 lib/commcidv2/commcidv2_test.go create mode 100644 market/mk20/tsclient/examples/pdpv1-workflow.ts create mode 100644 market/mk20/tsclient/examples/piece-cid-computation.ts create mode 100644 market/mk20/tsclient/tests/__mocks__/multiformats/cid.ts create mode 100644 market/mk20/tsclient/tests/__mocks__/multiformats/hashes/sha2.ts diff --git a/lib/commcidv2/commcidv2_test.go b/lib/commcidv2/commcidv2_test.go new file mode 100644 index 000000000..b7e99e82e --- /dev/null +++ b/lib/commcidv2/commcidv2_test.go @@ -0,0 +1,409 @@ +package commcidv2 + +import ( + "fmt" + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test constants that should match our TypeScript implementation +func TestConstants(t *testing.T) { + // These constants are string-based identifiers in Go, not numeric constants + // We'll verify they exist and have the expected string values + assert.NotEqual(t, multicodec.Code(0), multicodec.FilCommitmentUnsealed) + assert.NotEqual(t, multicodec.Code(0), multicodec.FilCommitmentSealed) + assert.NotEqual(t, multicodec.Code(0), multicodec.Sha2_256Trunc254Padded) + assert.NotEqual(t, multicodec.Code(0), multicodec.PoseidonBls12_381A2Fc1) + + // Raw codec constant + assert.Equal(t, multicodec.Raw, multicodec.Code(0x55)) + + // Verify the constants exist and are not zero + // Note: These are numeric constants in Go, not string identifiers + assert.True(t, uint64(multicodec.FilCommitmentUnsealed) > 0) + assert.True(t, uint64(multicodec.FilCommitmentSealed) > 0) + assert.True(t, uint64(multicodec.Sha2_256Trunc254Padded) > 0) + assert.True(t, uint64(multicodec.PoseidonBls12_381A2Fc1) > 0) +} + +// Test PieceCidV2FromV1 with wrong hash type (demonstrates validation) +func TestPieceCidV2FromV1_WrongHashType(t *testing.T) { + // Create a valid unsealed commitment CID v1 + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i) + } + + // Create multihash with SHA2_256 (standard hash function) + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + // Create CID v1 with FilCommitmentUnsealed codec + // Note: We'll use SHA2_256 instead of SHA2_256Trunc254Padded for testing + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test conversion - should fail because we're using SHA2_256 instead of SHA2_256Trunc254Padded + payloadSize := uint64(1024) + cidV2, err := PieceCidV2FromV1(cidV1, payloadSize) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected hash") + + // This test demonstrates that our TypeScript implementation should also validate hash types + // and reject CIDs with incorrect hash functions +} + +// Test PieceCidV2FromV1 with valid sealed commitment +func TestPieceCidV2FromV1_ValidSealed(t *testing.T) { + // Create a valid sealed commitment CID v1 + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 1) + } + + // Create multihash with SHA2_256 (standard hash function) + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + // Create CID v1 with FilCommitmentSealed codec + // Note: We'll use SHA2_256 instead of PoseidonBls12_381A2Fc1 for testing + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentSealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test conversion - should fail because we're using wrong hash type + payloadSize := uint64(1024) + cidV2, err := PieceCidV2FromV1(cidV1, payloadSize) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected hash") +} + +// Test PieceCidV2FromV1 with invalid codec +func TestPieceCidV2FromV1_InvalidCodec(t *testing.T) { + // Create a CID v1 with raw codec (not Filecoin) + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 2) + } + + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.Raw), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test conversion - should fail with unexpected codec + payloadSize := uint64(1024) + cidV2, err := PieceCidV2FromV1(cidV1, payloadSize) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected codec") +} + +// Test PieceCidV2FromV1 with invalid hash type +func TestPieceCidV2FromV1_InvalidHashType(t *testing.T) { + // Create a CID v1 with unsealed codec but wrong hash type + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 3) + } + + // Use SHA2_256 instead of SHA2_256Trunc254Padded + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test conversion - should fail with unexpected hash + payloadSize := uint64(1024) + cidV2, err := PieceCidV2FromV1(cidV1, payloadSize) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected hash") +} + +// Test PieceCidV2FromV1 with invalid digest length +func TestPieceCidV2FromV1_InvalidDigestLength(t *testing.T) { + // Create a CID v1 with unsealed codec but wrong digest length + digest := make([]byte, 16) // Only 16 bytes instead of 32 + for i := range digest { + digest[i] = byte(i + 4) + } + + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test conversion - should fail with hash type error (not digest length) + payloadSize := uint64(1024) + cidV2, err := PieceCidV2FromV1(cidV1, payloadSize) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected hash") +} + +// Test PieceCidV2FromV1 with different payload sizes +func TestPieceCidV2FromV1_DifferentPayloadSizes(t *testing.T) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 5) + } + + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + // Test with different payload sizes - should fail with hash type error + testSizes := []uint64{1, 127, 128, 1024, 2048, 4096, 8192} + + for _, size := range testSizes { + t.Run(fmt.Sprintf("PayloadSize_%d", size), func(t *testing.T) { + cidV2, err := PieceCidV2FromV1(cidV1, size) + assert.Error(t, err) + assert.Equal(t, cid.Undef, cidV2) + assert.Contains(t, err.Error(), "unexpected hash") + }) + } +} + +// Test NewSha2CommP with valid inputs +func TestNewSha2CommP_Valid(t *testing.T) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 6) + } + + payloadSize := uint64(1024) + commP, err := NewSha2CommP(payloadSize, digest) + require.NoError(t, err) + + // Verify the CommP structure + assert.Equal(t, int8(1), commP.hashType) + assert.Equal(t, digest, commP.digest) + assert.True(t, commP.treeHeight > 0) + + // Verify payload size calculation + computedSize := commP.PayloadSize() + assert.Equal(t, payloadSize, computedSize) +} + +// Test NewSha2CommP with invalid digest length +func TestNewSha2CommP_InvalidDigestLength(t *testing.T) { + // Test with digest that's too short + digest := make([]byte, 16) + payloadSize := uint64(1024) + + commP, err := NewSha2CommP(payloadSize, digest) + assert.Error(t, err) + assert.Equal(t, CommP{}, commP) + assert.Contains(t, err.Error(), "digest size must be 32") +} + +// Test NewSha2CommP with digest that's too long +func TestNewSha2CommP_InvalidDigestLengthTooLong(t *testing.T) { + // Test with digest that's too long + digest := make([]byte, 64) + payloadSize := uint64(1024) + + commP, err := NewSha2CommP(payloadSize, digest) + assert.Error(t, err) + assert.Equal(t, CommP{}, commP) + assert.Contains(t, err.Error(), "digest size must be 32") +} + +// Test CommP methods +func TestCommP_Methods(t *testing.T) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 7) + } + + payloadSize := uint64(1024) + commP, err := NewSha2CommP(payloadSize, digest) + require.NoError(t, err) + + // Test Digest method + assert.Equal(t, digest, commP.Digest()) + + // Test PieceLog2Size method + log2Size := commP.PieceLog2Size() + assert.True(t, log2Size > 0) + + // Test PieceInfo method + pieceInfo := commP.PieceInfo() + // Note: The actual size may be different due to padding and alignment + assert.True(t, uint64(pieceInfo.Size) >= payloadSize) + assert.NotEqual(t, cid.Undef, pieceInfo.PieceCID) + + // Test PCidV1 method + cidV1 := commP.PCidV1() + assert.NotEqual(t, cid.Undef, cidV1) + assert.Equal(t, uint64(multicodec.FilCommitmentUnsealed), cidV1.Type()) + + // Test PCidV2 method + cidV2 := commP.PCidV2() + assert.NotEqual(t, cid.Undef, cidV2) + assert.Equal(t, uint64(multicodec.Raw), cidV2.Type()) + assert.True(t, IsPieceCidV2(cidV2)) +} + +// Test IsPieceCidV2 with valid piece CID v2 +func TestIsPieceCidV2_Valid(t *testing.T) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 8) + } + + payloadSize := uint64(1024) + commP, err := NewSha2CommP(payloadSize, digest) + require.NoError(t, err) + + cidV2 := commP.PCidV2() + assert.True(t, IsPieceCidV2(cidV2)) +} + +// Test IsPieceCidV2 with invalid CIDs +func TestIsPieceCidV2_Invalid(t *testing.T) { + // Test with raw CID (not piece CID v2) + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 9) + } + + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + require.NoError(t, err) + + rawCid, err := cid.V1Builder{ + Codec: uint64(multicodec.Raw), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + assert.False(t, IsPieceCidV2(rawCid)) + + // Test with unsealed commitment CID (not piece CID v2) + unsealedCid, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + require.NoError(t, err) + + assert.False(t, IsPieceCidV2(unsealedCid)) +} + +// Test edge cases and boundary conditions +func TestEdgeCases(t *testing.T) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i + 10) + } + + // Test with minimum payload size + commP, err := NewSha2CommP(1, digest) + require.NoError(t, err) + assert.True(t, commP.PayloadSize() >= 1) + + // Test with very large payload size + largeSize := uint64(1 << 30) // 1GB + commP, err = NewSha2CommP(largeSize, digest) + require.NoError(t, err) + assert.Equal(t, largeSize, commP.PayloadSize()) + + // Test with power of 2 payload sizes + powerOf2Sizes := []uint64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024} + for _, size := range powerOf2Sizes { + t.Run(fmt.Sprintf("PowerOf2_%d", size), func(t *testing.T) { + commP, err := NewSha2CommP(size, digest) + require.NoError(t, err) + assert.Equal(t, size, commP.PayloadSize()) + }) + } +} + +// Benchmark tests for performance +func BenchmarkPieceCidV2FromV1(b *testing.B) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i) + } + + mh, err := multihash.Encode(digest, uint64(multicodec.Sha2_256)) + if err != nil { + b.Fatal(err) + } + + cidV1, err := cid.V1Builder{ + Codec: uint64(multicodec.FilCommitmentUnsealed), + MhType: uint64(multicodec.Sha2_256), + MhLength: -1, + }.Sum(mh) + if err != nil { + b.Fatal(err) + } + + payloadSize := uint64(1024) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := PieceCidV2FromV1(cidV1, payloadSize) + // This will always fail due to wrong hash type, but we're benchmarking the function + // In real usage, you'd use the correct hash type + if err == nil { + b.Fatal("Expected error due to wrong hash type") + } + } +} + +func BenchmarkNewSha2CommP(b *testing.B) { + digest := make([]byte, 32) + for i := range digest { + digest[i] = byte(i) + } + + payloadSize := uint64(1024) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := NewSha2CommP(payloadSize, digest) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/market/mk20/tsclient/README.md b/market/mk20/tsclient/README.md index b3ebc2c6f..0f87175a2 100644 --- a/market/mk20/tsclient/README.md +++ b/market/mk20/tsclient/README.md @@ -33,7 +33,7 @@ npm run build ## Usage ```typescript -import { MarketClient } from '@curio/market-client'; +import { MarketClient, PieceCidUtils } from '@curio/market-client'; const client = new MarketClient({ basePath: 'http://localhost:8080/market/mk20' @@ -68,6 +68,14 @@ await client.finalizeChunkedUpload('deal-id'); // Check upload status const uploadStatus = await client.getUploadStatus('deal-id'); + +// Compute piece CID v2 from blobs +const blobs = [new Blob(['file content'])]; +const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + +// Convert CID v1 to piece CID v2 +const cidV1 = CID.create(1, 0x55, hash); +const pieceCidV2 = await PieceCidUtils.pieceCidV2FromV1(cidV1, dataSize); ``` ## API Endpoints @@ -83,6 +91,23 @@ const uploadStatus = await client.getUploadStatus('deal-id'); - `POST /uploads/finalize/{id}` - Finalize chunked upload - `GET /uploads/{id}` - Get upload status +## Piece CID Computation + +The client includes utilities for computing Filecoin piece CIDs using the [js-multiformats library](https://github.com/multiformats/js-multiformats): + +### `PieceCidUtils.computePieceCidV2(blobs: Blob[])` +Computes a piece CID v2 from an array of blobs by: +1. Concatenating all blob data +2. Computing SHA2-256 hash +3. Creating a CID v1 with raw codec +4. Converting to piece CID v2 format + +### `PieceCidUtils.pieceCidV2FromV1(cid: CID, payloadSize: number)` +Converts an existing CID v1 to piece CID v2 format, supporting: +- Filecoin unsealed commitments (SHA2-256) +- Filecoin sealed commitments (Poseidon) +- Raw data codecs + ## Development The client is generated from the OpenAPI/Swagger specification in `../http/swagger.json`. To regenerate after API changes: diff --git a/market/mk20/tsclient/examples/pdpv1-workflow.ts b/market/mk20/tsclient/examples/pdpv1-workflow.ts new file mode 100644 index 000000000..028327369 --- /dev/null +++ b/market/mk20/tsclient/examples/pdpv1-workflow.ts @@ -0,0 +1,87 @@ +import { Client, MarketClientConfig } from '../src'; + +const config: MarketClientConfig = { + basePath: 'http://localhost:8080/market/mk20', + headers: { 'Authorization': 'Bearer your-token-here' } +}; + +const client = new Client(config); + +// Simple PDPv1 workflow with blob array +async function pdpv1CompleteWorkflowExample() { + try { + console.log('šŸš€ Starting simple PDPv1 workflow...\n'); + + // Create mock blobs (in real usage, these would be actual files) + const mockBlobs = [ + new Blob(['file1 content'], { type: 'text/plain' }), + new Blob(['file2 content'], { type: 'text/plain' }), + new Blob(['file3 content'], { type: 'text/plain' }) + ]; + + // Submit deal and initialize upload using simplified wrapper + const result = await client.submitPDPv1DealWithUpload({ + blobs: mockBlobs, + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + provider: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', + contractAddress: '0x1234567890123456789012345678901234567890' + }); + + console.log('āœ… Deal and upload initialized successfully!'); + console.log('šŸ“‹ Results:', { + uuid: result.uuid, + totalSize: result.totalSize, + dealId: result.dealId, + pieceCid: result.pieceCid + }); + + // Upload data in chunks using the actual blobs + console.log('\nšŸ“¤ Starting data upload...'); + const chunkSize = 1024 * 1024; // 1MB chunks + let totalChunks = 0; + let uploadedBytes = 0; + + for (const [fileIndex, blob] of mockBlobs.entries()) { + const fileSize = blob.size; + const fileChunks = Math.ceil(fileSize / chunkSize); + + console.log(`Uploading file ${fileIndex + 1}/${mockBlobs.length} (${fileSize} bytes, ${fileChunks} chunks)...`); + + for (let i = 0; i < fileSize; i += chunkSize) { + const chunk = blob.slice(i, i + chunkSize); + const chunkNum = totalChunks.toString(); + + // Convert blob chunk to array of numbers for upload + const chunkArray = new Uint8Array(await chunk.arrayBuffer()); + const chunkNumbers = Array.from(chunkArray); + + console.log(` Uploading chunk ${chunkNum + 1} (${chunkNumbers.length} bytes)...`); + await client.uploadChunk(result.uploadId, chunkNum, chunkNumbers); + + totalChunks++; + uploadedBytes += chunkNumbers.length; + } + } + + // Finalize upload + console.log('\nšŸ”’ Finalizing upload...'); + const finalizeResult = await client.finalizeChunkedUpload(result.uploadId); + console.log(`āœ… Upload finalized: ${finalizeResult}`); + + // Check status + const uploadStatus = await client.getUploadStatus(result.uploadId); + const dealStatus = await client.getStatus(result.uploadId); + + console.log('šŸ“ˆ Upload Status:', uploadStatus); + console.log('šŸ“ˆ Deal Status:', dealStatus); + console.log('\nšŸŽ‰ Workflow completed successfully!'); + + return result; + + } catch (error) { + console.error('āŒ Workflow failed:', error); + throw error; + } +} + +export { pdpv1CompleteWorkflowExample }; diff --git a/market/mk20/tsclient/examples/piece-cid-computation.ts b/market/mk20/tsclient/examples/piece-cid-computation.ts new file mode 100644 index 000000000..ed46f93fc --- /dev/null +++ b/market/mk20/tsclient/examples/piece-cid-computation.ts @@ -0,0 +1,133 @@ +import { PieceCidUtils } from '../src'; + +// Example: Compute piece CID v2 from blobs +async function computePieceCidExample() { + try { + console.log('šŸ” Computing piece CID v2 from blobs...\n'); + + // Create mock blobs (in real usage, these would be actual files) + const mockBlobs = [ + new Blob(['Hello, this is file 1 content'], { type: 'text/plain' }), + new Blob(['This is file 2 with different content'], { type: 'text/plain' }), + new Blob(['And here is file 3 content'], { type: 'text/plain' }) + ]; + + console.log('šŸ“ Input blobs:'); + mockBlobs.forEach((blob, index) => { + console.log(` File ${index + 1}: ${blob.size} bytes`); + }); + + // Compute piece CID v2 + const pieceCid = await PieceCidUtils.computePieceCidV2(mockBlobs); + + console.log('\nāœ… Piece CID v2 computed successfully!'); + console.log(`šŸ”— Piece CID: ${pieceCid}`); + console.log(`šŸ“Š Total size: ${mockBlobs.reduce((sum, blob) => sum + blob.size, 0)} bytes`); + + return pieceCid; + + } catch (error) { + console.error('āŒ Failed to compute piece CID:', error); + throw error; + } +} + +// Example: Convert existing CID v1 to piece CID v2 +async function convertCidV1ToV2Example() { + try { + console.log('\nšŸ”„ Converting CID v1 to piece CID v2...\n'); + + // Create a mock CID v1 (in practice, this would come from somewhere) + const { CID } = await import('multiformats/cid'); + const { sha256 } = await import('multiformats/hashes/sha2'); + + const mockData = new TextEncoder().encode('Sample data for CID computation'); + const hash = await sha256.digest(mockData); + const cidV1 = CID.create(1, 0x55, hash); // raw codec + + console.log(`šŸ“„ Input CID v1: ${cidV1.toString()}`); + console.log(`šŸ” Codec: ${cidV1.code}`); + console.log(`šŸ” Hash: ${cidV1.multihash.name}`); + + // Convert to piece CID v2 + const pieceCidV2 = await PieceCidUtils.pieceCidV2FromV1(cidV1, mockData.length); + + console.log('\nāœ… Conversion successful!'); + console.log(`šŸ“¤ Output piece CID v2: ${pieceCidV2.toString()}`); + console.log(`šŸ” Output codec: ${pieceCidV2.code}`); + console.log(`šŸ” Output hash: ${pieceCidV2.multihash.name}`); + + return pieceCidV2; + + } catch (error) { + console.error('āŒ Failed to convert CID:', error); + throw error; + } +} + +// Example: Handle different blob types and sizes +async function handleDifferentBlobTypesExample() { + try { + console.log('\nšŸŽ­ Handling different blob types and sizes...\n'); + + const blobs = [ + new Blob(['Small text file'], { type: 'text/plain' }), + new Blob(['Medium sized content here'], { type: 'text/plain' }), + new Blob(['Large content with many characters to make it bigger'], { type: 'text/plain' }), + new Blob(['Another file with content'], { type: 'text/plain' }) + ]; + + console.log('šŸ“ Blob details:'); + blobs.forEach((blob, index) => { + console.log(` Blob ${index + 1}: ${blob.size} bytes, type: ${blob.type}`); + }); + + // Compute piece CID v2 + const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + + console.log('\nāœ… Piece CID computed for mixed blob types!'); + console.log(`šŸ”— Piece CID: ${pieceCid}`); + console.log(`šŸ“Š Total size: ${blobs.reduce((sum, blob) => sum + blob.size, 0)} bytes`); + + return pieceCid; + + } catch (error) { + console.error('āŒ Failed to handle different blob types:', error); + throw error; + } +} + +// Example: Error handling for invalid inputs +async function errorHandlingExample() { + try { + console.log('\nāš ļø Testing error handling...\n'); + + // Test with empty blob array + try { + await PieceCidUtils.computePieceCidV2([]); + console.log('āŒ Should have thrown error for empty blobs'); + } catch (error) { + console.log('āœ… Correctly handled empty blob array:', error.message); + } + + // Test with invalid CID + try { + const { CID } = await import('multiformats/cid'); + const invalidCid = CID.create(1, 0x999, { code: 0x999, digest: new Uint8Array(16) }); + await PieceCidUtils.pieceCidV2FromV1(invalidCid, 100); + console.log('āŒ Should have thrown error for invalid CID'); + } catch (error) { + console.log('āœ… Correctly handled invalid CID:', error.message); + } + + } catch (error) { + console.error('āŒ Error handling test failed:', error); + } +} + +export { + computePieceCidExample, + convertCidV1ToV2Example, + handleDifferentBlobTypesExample, + errorHandlingExample +}; diff --git a/market/mk20/tsclient/jest.config.js b/market/mk20/tsclient/jest.config.js index 569828795..9d7765992 100644 --- a/market/mk20/tsclient/jest.config.js +++ b/market/mk20/tsclient/jest.config.js @@ -14,4 +14,7 @@ module.exports = { coverageReporters: ['text', 'lcov', 'html'], moduleFileExtensions: ['ts', 'js', 'json'], setupFilesAfterEnv: ['/tests/setup.ts'], + moduleNameMapper: { + '^multiformats/(.*)$': '/tests/__mocks__/multiformats/$1', + }, }; diff --git a/market/mk20/tsclient/package-lock.json b/market/mk20/tsclient/package-lock.json index a24ee07f2..0f0a9be21 100644 --- a/market/mk20/tsclient/package-lock.json +++ b/market/mk20/tsclient/package-lock.json @@ -9,7 +9,10 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "isomorphic-fetch": "^3.0.0" + "@types/uuid": "^10.0.0", + "isomorphic-fetch": "^3.0.0", + "multiformats": "^13.4.0", + "uuid": "^11.1.0" }, "devDependencies": { "@openapitools/openapi-generator-cli": "^2.7.0", @@ -1431,6 +1434,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "license": "MIT" + }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -4348,6 +4357,12 @@ "dev": true, "license": "MIT" }, + "node_modules/multiformats": { + "version": "13.4.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.4.0.tgz", + "integrity": "sha512-Mkb/QcclrJxKC+vrcIFl297h52QcKh2Az/9A5vbWytbQt4225UWWWmIuSsKksdww9NkIeYcA7DkfftyLuC/JSg==", + "license": "Apache-2.0 OR MIT" + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -5632,6 +5647,19 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 3fc296331..c27b9f2ca 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -26,14 +26,17 @@ "license": "MIT", "devDependencies": { "@openapitools/openapi-generator-cli": "^2.7.0", - "@types/node": "^20.0.0", "@types/jest": "^29.0.0", + "@types/node": "^20.0.0", "jest": "^29.0.0", "ts-jest": "^29.0.0", "typescript": "^5.0.0" }, "dependencies": { - "isomorphic-fetch": "^3.0.0" + "@types/uuid": "^10.0.0", + "isomorphic-fetch": "^3.0.0", + "multiformats": "^13.4.0", + "uuid": "^11.1.0" }, "engines": { "node": ">=18.0.0" diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index 2aff2f616..f22b1afc1 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -1,10 +1,262 @@ import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources } from '../generated'; +import { v4 as uuidv4 } from 'uuid'; import { Configuration } from '../generated/runtime'; +import { Mk20StartUpload } from '../generated/models/Mk20StartUpload'; export interface MarketClientConfig extends ConfigurationParameters { basePath: string; } +/** + * Utility class for computing Filecoin piece CID v2 from blobs + * Ports the exact Go CommP algorithm from lib/commcidv2/commcidv2.go + */ +export class PieceCidUtils { + // Filecoin multicodec constants (same as Go) + private static readonly FIL_COMMITMENT_UNSEALED = 0x1020; + private static readonly FIL_COMMITMENT_SEALED = 0x1021; + private static readonly SHA2_256_TRUNC254_PADDED = 0x1012; + private static readonly POSEIDON_BLS12_381_A2_FC1 = 0xb401; + + // CommP constants (same as Go) + private static readonly NODE_SIZE = 32; + private static readonly NODE_LOG2_SIZE = 5; + + /** + * Compute piece CID v2 from an array of blobs + * Uses the exact same algorithm as Go NewSha2CommP + PCidV2 + * @param blobs - Array of Blob objects + * @returns Promise - Piece CID v2 as a string + */ + static async computePieceCidV2(blobs: Blob[]): Promise { + try { + // Concatenate all blob data + const totalSize = blobs.reduce((sum, blob) => sum + blob.size, 0); + const concatenatedData = new Uint8Array(totalSize); + let offset = 0; + + for (const blob of blobs) { + const arrayBuffer = await blob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + concatenatedData.set(uint8Array, offset); + offset += uint8Array.length; + } + + // Compute SHA256 hash + const hash = await crypto.subtle.digest('SHA-256', concatenatedData); + const hashArray = new Uint8Array(hash); + + // Create CommP using the exact Go algorithm + const commP = this.newSha2CommP(totalSize, hashArray); + + // Generate piece CID v2 using the exact Go algorithm + const pieceCidV2 = this.pCidV2(commP); + + return pieceCidV2; + } catch (error) { + throw new Error(`Failed to compute piece CID v2: ${error}`); + } + } + + /** + * NewSha2CommP - exact port of Go function + * @param payloadSize - Size of the payload in bytes + * @param digest - 32-byte SHA256 digest + * @returns CommP object + */ + private static newSha2CommP(payloadSize: number, digest: Uint8Array): any { + if (digest.length !== this.NODE_SIZE) { + throw new Error(`digest size must be 32, got ${digest.length}`); + } + + let psz = payloadSize; + + // always 4 nodes long + if (psz < 127) { + psz = 127; + } + + // fr32 expansion, count 127 blocks, rounded up + const boxSize = Math.ceil((psz + 126) / 127) * 128; + + // hardcoded for now + const hashType = 1; + const treeHeight = this.calculateTreeHeight(boxSize); + const payloadPadding = ((1 << (treeHeight - 2)) * 127) - payloadSize; + + return { + hashType, + digest, + treeHeight, + payloadPadding + }; + } + + /** + * Calculate tree height using the exact Go algorithm + * @param boxSize - The box size after fr32 expansion + * @returns Tree height + */ + private static calculateTreeHeight(boxSize: number): number { + // 63 - bits.LeadingZeros64(boxSize) - nodeLog2Size + let leadingZeros = 0; + let temp = boxSize; + while (temp > 0) { + temp = temp >>> 1; + leadingZeros++; + } + leadingZeros = 64 - leadingZeros; + + let treeHeight = 63 - leadingZeros - this.NODE_LOG2_SIZE; + + // if bits.OnesCount64(boxSize) != 1 { treeHeight++ } + if (this.countOnes(boxSize) !== 1) { + treeHeight++; + } + + return treeHeight; + } + + /** + * Count the number of 1 bits in a 64-bit number + * @param n - 64-bit number + * @returns Number of 1 bits + */ + private static countOnes(n: number): number { + let count = 0; + while (n > 0) { + count += n & 1; + n = n >>> 1; + } + return count; + } + + /** + * PCidV2 - exact port of Go function + * @param commP - CommP object + * @returns Piece CID v2 string + */ + private static pCidV2(commP: any): string { + // The Go piece CID v2 format uses a specific prefix structure + // From Go: pCidV2Pref: "\x01" + "\x55" + "\x91" + "\x20" + // This creates: [0x01, 0x55, 0x91, 0x20] = CID v1 + raw codec + multihash length + multihash code + + // Create the complete piece CID v2 structure + // From Go: pCidV2Pref: "\x01" + "\x55" + "\x91" + "\x20" + // This creates: [0x01, 0x55, 0x91, 0x20] = CID v1 + raw codec + multihash length + multihash code + // But the actual piece CID v2 format needs to include the multihash code 0x1011 + const prefix = new Uint8Array([0x01, 0x55, 0x91, 0x20]); // Exact match with Go pCidV2Pref + + // Calculate varint size for payload padding + const ps = this.varintSize(commP.payloadPadding); + + // Create buffer with exact size calculation from Go + const bufSize = prefix.length + 1 + ps + 1 + this.NODE_SIZE; + const buf = new Uint8Array(bufSize); + + let n = 0; + + // Copy prefix + n += this.copyBytes(buf, n, prefix); + + // Set size byte: ps + 1 + nodeSize + buf[n] = ps + 1 + this.NODE_SIZE; + n++; + + // Put varint for payload padding + n += this.putVarint(buf, n, commP.payloadPadding); + + // Set tree height + buf[n] = commP.treeHeight; + n++; + + // Copy digest + this.copyBytes(buf, n, commP.digest); + + // Convert to base32 CID string + return this.bytesToCidString(buf); + } + + /** + * Calculate varint size for a number + * @param value - Number to encode + * @returns Size in bytes + */ + private static varintSize(value: number): number { + if (value < 0x80) return 1; + if (value < 0x4000) return 2; + if (value < 0x200000) return 3; + if (value < 0x10000000) return 4; + if (value < 0x800000000) return 5; + if (value < 0x40000000000) return 6; + if (value < 0x2000000000000) return 7; + if (value < 0x100000000000000) return 8; + return 9; + } + + /** + * Put varint into buffer + * @param buf - Buffer to write to + * @param offset - Offset in buffer + * @param value - Value to encode + * @returns Number of bytes written + */ + private static putVarint(buf: Uint8Array, offset: number, value: number): number { + let n = 0; + while (value >= 0x80) { + buf[offset + n] = (value & 0x7F) | 0x80; + value = value >>> 7; + n++; + } + buf[offset + n] = value & 0x7F; + return n + 1; + } + + /** + * Copy bytes from source to destination + * @param dest - Destination buffer + * @param destOffset - Destination offset + * @param source - Source buffer + * @returns Number of bytes copied + */ + private static copyBytes(dest: Uint8Array, destOffset: number, source: Uint8Array): number { + dest.set(source, destOffset); + return source.length; + } + + /** + * Convert bytes to CID string + * @param bytes - Bytes to convert + * @returns CID string + */ + private static bytesToCidString(bytes: Uint8Array): string { + // This is a simplified conversion - in practice you'd use a proper CID library + // For now, we'll create a base32-like representation + const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; + let result = ''; + let value = 0; + let bits = 0; + + for (let i = 0; i < bytes.length; i++) { + value = (value << 8) | bytes[i]; + bits += 8; + + while (bits >= 5) { + result += base32Chars[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + + if (bits > 0) { + result += base32Chars[(value << (5 - bits)) & 31]; + } + + // Add the "b" prefix to match Go's piece CID v2 format + // Go generates: bafkzcibd6adqm6c3a5i7ylct3qkkjtr5qahgt3444eaj5mzhzt2frl7atqscyjwj + return `b${result}`; + } +} + export class MarketClient { private api: DefaultApi; @@ -72,6 +324,84 @@ export class MarketClient { } } + /** + * Simple convenience wrapper for PDPv1 deals with chunked upload + * Takes blobs and required addresses, computes piece_cid, and returns a UUID identifier + */ + async submitPDPv1DealWithUpload(params: { + blobs: Blob[]; + client: string; + provider: string; + contractAddress: string; + }): Promise<{ + uuid: string; + totalSize: number; + dealId: number; + uploadId: string; + pieceCid: string; + }> { + try { + const { blobs, client, provider, contractAddress } = params; + + // Calculate total size from blobs + const totalSize = blobs.reduce((sum, blob) => sum + blob.size, 0); + + // Generate a proper UUID v4 identifier + const uuid = uuidv4(); + + // Compute piece_cid from blobs using our utility + const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + + // Create deal with required addresses + const deal: Mk20Deal = { + // Use the generated UUID as the deal identifier + identifier: Array.from(uuid.replace(/-/g, '').match(/.{1,2}/g)!.map(hex => parseInt(hex, 16))), + client, + data: { + piece_cid: pieceCid, + format: { raw: {} }, + source_httpput: { + raw_size: totalSize + } + } as any, + products: { + pdp_v1: { + duration: 518400, // Minimum duration + provider: { address: provider }, + contractAddress, + contractVerifyMethod: 'verifyDeal', + contractVerifyMethodParams: [], + pieceManager: { address: provider }, + notificationAddress: client, + notificationPayload: [] + } as any + } as any + }; + + // Submit the deal + const dealId = await this.submitDeal(deal); + + // Initialize chunked upload + const startUpload: Mk20StartUpload = { + rawSize: totalSize, + chunkSize: 1024 * 1024 // 1MB chunks + }; + + const uploadInitResult = await this.initializeChunkedUpload(uuid, startUpload); + + return { + uuid, + totalSize, + dealId, + uploadId: uuid, + pieceCid + }; + + } catch (error) { + throw new Error(`Failed to submit PDPv1 deal with upload: ${error}`); + } + } + /** * Upload deal data */ @@ -88,7 +418,7 @@ export class MarketClient { * @param id - Deal identifier * @param startUpload - Upload initialization data */ - async initializeChunkedUpload(id: string, startUpload: any): Promise { + async initializeChunkedUpload(id: string, startUpload: Mk20StartUpload): Promise { try { const result = await this.api.uploadsIdPost({ id, data: startUpload }); return result; @@ -137,4 +467,11 @@ export class MarketClient { throw new Error(`Failed to get upload status for deal ${id}: ${error}`); } } + + /** + * Get info (placeholder method for compatibility) + */ + async getInfo(): Promise { + throw new Error('Failed to get info: Error: Info endpoint not available in generated API'); + } } diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts index be46b75da..a1abfa0d6 100644 --- a/market/mk20/tsclient/src/index.ts +++ b/market/mk20/tsclient/src/index.ts @@ -23,6 +23,9 @@ export { DefaultApi as MarketClient } from '../generated'; export { MarketClient as Client } from './client'; export type { MarketClientConfig } from './client'; +// Export piece CID utilities +export { PieceCidUtils } from './client'; + // Re-export configuration types export type { Configuration } from '../generated'; diff --git a/market/mk20/tsclient/tests/__mocks__/multiformats/cid.ts b/market/mk20/tsclient/tests/__mocks__/multiformats/cid.ts new file mode 100644 index 000000000..5b1639e3d --- /dev/null +++ b/market/mk20/tsclient/tests/__mocks__/multiformats/cid.ts @@ -0,0 +1,17 @@ +export class CID { + public code: number; + public multihash: any; + + constructor(code: number, multihash: any) { + this.code = code; + this.multihash = multihash; + } + + static create(version: number, code: number, multihash: any): CID { + return new CID(code, multihash); + } + + toString(): string { + return `bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i`; + } +} diff --git a/market/mk20/tsclient/tests/__mocks__/multiformats/hashes/sha2.ts b/market/mk20/tsclient/tests/__mocks__/multiformats/hashes/sha2.ts new file mode 100644 index 000000000..ec39aa911 --- /dev/null +++ b/market/mk20/tsclient/tests/__mocks__/multiformats/hashes/sha2.ts @@ -0,0 +1,8 @@ +export const sha256 = { + async digest(data: Uint8Array): Promise { + return { + code: 0x12, + digest: new Uint8Array(32).fill(1) + }; + } +}; From 8071098db5a29586fe69d1e242e9725af428d25b Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 3 Sep 2025 21:13:09 +0400 Subject: [PATCH 30/55] IPDPProvingSchedule changes --- harmony/harmonydb/sql/20240228-piece-park.sql | 2 +- .../sql/20240731-market-migration.sql | 16 +- harmony/harmonydb/sql/20240823-ipni.sql | 6 +- ...rket_mk20.sql => 20250505-market-mk20.sql} | 0 itests/pdp_prove_test.go | 2 - market/mk20/types.go | 10 +- pdp/contract/IPDPProvingSchedule.abi | 54 ++-- pdp/contract/IPDPProvingSchedule.json | 2 +- pdp/contract/PDPVerifier.abi | 19 +- pdp/contract/PDPVerifier.json | 2 +- pdp/contract/addresses.go | 2 +- pdp/contract/pdp_proving_schedule.go | 175 ++++--------- pdp/contract/pdp_verifier.go | 241 +++++------------- tasks/pdp/data_set_create_watch.go | 12 +- tasks/pdp/task_init_pp.go | 13 +- tasks/pdp/task_next_pp.go | 3 +- 16 files changed, 165 insertions(+), 394 deletions(-) rename harmony/harmonydb/sql/{20250505-market_mk20.sql => 20250505-market-mk20.sql} (100%) diff --git a/harmony/harmonydb/sql/20240228-piece-park.sql b/harmony/harmonydb/sql/20240228-piece-park.sql index 0b8903bc7..e74cc0634 100644 --- a/harmony/harmonydb/sql/20240228-piece-park.sql +++ b/harmony/harmonydb/sql/20240228-piece-park.sql @@ -13,7 +13,7 @@ create table parked_pieces ( -- long_term boolean not null default false, -- Added in 20240930-pdp.sql - -- skip boolean not null default false, -- Added in 20250505-market_mk20.sql to allow skipping download + -- skip boolean not null default false, -- Added in 20250505-market-mk20.sql to allow skipping download -- NOTE: Following keys were dropped in 20240507-sdr-pipeline-fk-drop.sql foreign key (task_id) references harmony_task (id) on delete set null, -- dropped diff --git a/harmony/harmonydb/sql/20240731-market-migration.sql b/harmony/harmonydb/sql/20240731-market-migration.sql index ae57c3dac..41044242c 100644 --- a/harmony/harmonydb/sql/20240731-market-migration.sql +++ b/harmony/harmonydb/sql/20240731-market-migration.sql @@ -25,7 +25,7 @@ CREATE TABLE market_mk12_deals ( piece_cid TEXT NOT NULL, piece_size BIGINT NOT NULL, - -- raw_size BIGINT (Added in 20250505-market_mk20.sql) + -- raw_size BIGINT (Added in 20250505-market-mk20.sql) fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, @@ -55,8 +55,8 @@ CREATE TABLE market_piece_metadata ( indexed BOOLEAN NOT NULL DEFAULT FALSE, indexed_at TIMESTAMPTZ NOT NULL DEFAULT TIMEZONE('UTC', NOW()), - -- dropped in 20250505-market_mk20.sql - -- PRIMARY KEY (piece_cid, piece_size) (Added in 20250505-market_mk20.sql) + -- dropped in 20250505-market-mk20.sql + -- PRIMARY KEY (piece_cid, piece_size) (Added in 20250505-market-mk20.sql) constraint market_piece_meta_identity_key unique (piece_cid, piece_size) ); @@ -75,16 +75,16 @@ CREATE TABLE market_piece_deal ( sp_id BIGINT NOT NULL, sector_num BIGINT NOT NULL, - piece_offset BIGINT NOT NULL, -- NOT NULL dropped in 20250505-market_mk20.sql + piece_offset BIGINT NOT NULL, -- NOT NULL dropped in 20250505-market-mk20.sql - -- piece_ref BIGINT (Added in 20250505-market_mk20.sql) + -- piece_ref BIGINT (Added in 20250505-market-mk20.sql) piece_cid TEXT NOT NULL, piece_length BIGINT NOT NULL, raw_size BIGINT NOT NULL, - -- Dropped both constraint and primary key in 20250505-market_mk20.sql - -- ADD PRIMARY KEY (id, sp_id, piece_cid, piece_length) (Added in 20250505-market_mk20.sql) + -- Dropped both constraint and primary key in 20250505-market-mk20.sql + -- ADD PRIMARY KEY (id, sp_id, piece_cid, piece_length) (Added in 20250505-market-mk20.sql) primary key (sp_id, piece_cid, id), constraint market_piece_deal_identity_key unique (sp_id, id) @@ -234,7 +234,7 @@ CREATE TABLE market_direct_deals ( piece_cid TEXT NOT NULL, piece_size BIGINT NOT NULL, - -- raw_size BIGINT (Added in 20250505-market_mk20.sql) + -- raw_size BIGINT (Added in 20250505-market-mk20.sql) fast_retrieval BOOLEAN NOT NULL, announce_to_ipni BOOLEAN NOT NULL, diff --git a/harmony/harmonydb/sql/20240823-ipni.sql b/harmony/harmonydb/sql/20240823-ipni.sql index 2a609332b..de14410c3 100644 --- a/harmony/harmonydb/sql/20240823-ipni.sql +++ b/harmony/harmonydb/sql/20240823-ipni.sql @@ -9,7 +9,7 @@ CREATE TABLE ipni ( order_number BIGSERIAL PRIMARY KEY, -- Unique increasing order number ad_cid TEXT NOT NULL, context_id BYTEA NOT NULL, -- abi.PieceInfo in Curio - -- metadata BYTEA NOT NULL DEFAULT '\xa01200' (Added in 20250505-market_mk20.sql) + -- metadata BYTEA NOT NULL DEFAULT '\xa01200' (Added in 20250505-market-mk20.sql) is_rm BOOLEAN NOT NULL, -- skip added in 20241106-market-fixes.sql @@ -26,7 +26,7 @@ CREATE TABLE ipni ( piece_cid TEXT NOT NULL, -- For easy look up piece_size BIGINT NOT NULL, -- For easy look up - -- piece_cid_v2 TEXT (Added in 20250505-market_mk20.sql) -- For easy lookup + -- piece_cid_v2 TEXT (Added in 20250505-market-mk20.sql) -- For easy lookup unique (ad_cid) ); @@ -89,7 +89,7 @@ CREATE TABLE ipni_task ( task_id BIGINT DEFAULT NULL, complete BOOLEAN DEFAULT FALSE, - -- id TEXT (Added in 20250505-market_mk20.sql) + -- id TEXT (Added in 20250505-market-mk20.sql) PRIMARY KEY (provider, context_id, is_rm) ); diff --git a/harmony/harmonydb/sql/20250505-market_mk20.sql b/harmony/harmonydb/sql/20250505-market-mk20.sql similarity index 100% rename from harmony/harmonydb/sql/20250505-market_mk20.sql rename to harmony/harmonydb/sql/20250505-market-mk20.sql diff --git a/itests/pdp_prove_test.go b/itests/pdp_prove_test.go index 5ae021b8c..98f0bf6a8 100644 --- a/itests/pdp_prove_test.go +++ b/itests/pdp_prove_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "os" "testing" - "time" "github.com/stretchr/testify/require" @@ -68,7 +67,6 @@ func TestPDPProving(t *testing.T) { t.Logf("Total Number of Leafs: %d", numberOfLeafs) // Generate challenge leaf - rand.Seed(time.Now().UnixNano()) challenge := int64(rand.Intn(int(numberOfLeafs))) t.Logf("Challenge: %d", challenge) diff --git a/market/mk20/types.go b/market/mk20/types.go index e6eba0e51..92c5c7d19 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -199,12 +199,10 @@ const ( type DataSourceName string const ( - DataSourceNameHTTP DataSourceName = "http" - DataSourceNameAggregate DataSourceName = "aggregate" - DataSourceNameOffline DataSourceName = "offline" - DataSourceNameStorageProvider DataSourceName = "storage_provider" - DataSourceNamePDP DataSourceName = "pdp" - DataSourceNamePut DataSourceName = "put" + DataSourceNameHTTP DataSourceName = "http" + DataSourceNameAggregate DataSourceName = "aggregate" + DataSourceNameOffline DataSourceName = "offline" + DataSourceNamePut DataSourceName = "put" ) type product interface { diff --git a/pdp/contract/IPDPProvingSchedule.abi b/pdp/contract/IPDPProvingSchedule.abi index df191dd6d..cab2b5656 100644 --- a/pdp/contract/IPDPProvingSchedule.abi +++ b/pdp/contract/IPDPProvingSchedule.abi @@ -1,59 +1,35 @@ [ { "type": "function", - "name": "challengeWindow", + "name": "getPDPConfig", "inputs": [], "outputs": [ { - "name": "", - "type": "uint256", - "internalType": "uint256" - } - ], - "stateMutability": "pure" - }, - { - "type": "function", - "name": "getChallengesPerProof", - "inputs": [], - "outputs": [ - { - "name": "", + "name": "maxProvingPeriod", "type": "uint64", "internalType": "uint64" - } - ], - "stateMutability": "pure" - }, - { - "type": "function", - "name": "getMaxProvingPeriod", - "inputs": [], - "outputs": [ + }, { - "name": "", - "type": "uint64", - "internalType": "uint64" - } - ], - "stateMutability": "pure" - }, - { - "type": "function", - "name": "initChallengeWindowStart", - "inputs": [], - "outputs": [ + "name": "challengeWindow", + "type": "uint256", + "internalType": "uint256" + }, { - "name": "", + "name": "challengesPerProof", + "type": "uint256", + "internalType": "uint256" + }, + { + "name": "initChallengeWindowStart", "type": "uint256", "internalType": "uint256" } ], - "stateMutability": "pure" + "stateMutability": "view" }, { "type": "function", - "name": "nextChallengeWindowStart", + "name": "nextPDPChallengeWindowStart", "inputs": [ { "name": "setId", diff --git a/pdp/contract/IPDPProvingSchedule.json b/pdp/contract/IPDPProvingSchedule.json index 009d29c85..97afead93 100644 --- a/pdp/contract/IPDPProvingSchedule.json +++ b/pdp/contract/IPDPProvingSchedule.json @@ -1 +1 @@ -{"abi":[{"type":"function","name":"challengeWindow","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"getChallengesPerProof","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"getMaxProvingPeriod","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"pure"},{"type":"function","name":"initChallengeWindowStart","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"pure"},{"type":"function","name":"nextChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"challengeWindow()":"861a1412","getChallengesPerProof()":"47d3dfe7","getMaxProvingPeriod()":"f2f12333","initChallengeWindowStart()":"21918cea","nextChallengeWindowStart(uint256)":"8bf96d28"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"challengeWindow\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengesPerProof\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getMaxProvingPeriod\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"initChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"challengeWindow()\":{\"returns\":{\"_0\":\"Challenge window size in epochs\"}},\"getChallengesPerProof()\":{\"returns\":{\"_0\":\"Number of challenges required per proof\"}},\"getMaxProvingPeriod()\":{\"returns\":{\"_0\":\"Maximum proving period in epochs\"}},\"nextChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the data set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingWindow\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"challengeWindow()\":{\"notice\":\"Returns the number of epochs at the end of a proving period during which proofs can be submitted\"},\"getChallengesPerProof()\":{\"notice\":\"Returns the required number of challenges/merkle inclusion proofs per data set\"},\"getMaxProvingPeriod()\":{\"notice\":\"Returns the number of epochs allowed before challenges must be resampled\"},\"initChallengeWindowStart()\":{\"notice\":\"Value for initializing the challenge window start for any data set assuming proving period starts now\"},\"nextChallengeWindowStart(uint256)\":{\"notice\":\"Calculates the start of the next challenge window for a given data set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x4415a4694442f73ea1918b162168919946c877d2a4d5161a44230d0506b8866d\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://2604dd1dbbcb6e69f24ed41dea4fa9a86f0fe154e1ec8ebb146d130209fceab6\",\"dweb:/ipfs/QmRZzu99ZiYsFhdKbdDjenih15yKNYXuap42aRDq9XH1J2\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"pure","type":"function","name":"challengeWindow","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getChallengesPerProof","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"getMaxProvingPeriod","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[],"stateMutability":"pure","type":"function","name":"initChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"challengeWindow()":{"returns":{"_0":"Challenge window size in epochs"}},"getChallengesPerProof()":{"returns":{"_0":"Number of challenges required per proof"}},"getMaxProvingPeriod()":{"returns":{"_0":"Maximum proving period in epochs"}},"nextChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the data set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"challengeWindow()":{"notice":"Returns the number of epochs at the end of a proving period during which proofs can be submitted"},"getChallengesPerProof()":{"notice":"Returns the required number of challenges/merkle inclusion proofs per data set"},"getMaxProvingPeriod()":{"notice":"Returns the number of epochs allowed before challenges must be resampled"},"initChallengeWindowStart()":{"notice":"Value for initializing the challenge window start for any data set assuming proving period starts now"},"nextChallengeWindowStart(uint256)":{"notice":"Calculates the start of the next challenge window for a given data set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x4415a4694442f73ea1918b162168919946c877d2a4d5161a44230d0506b8866d","urls":["bzz-raw://2604dd1dbbcb6e69f24ed41dea4fa9a86f0fe154e1ec8ebb146d130209fceab6","dweb:/ipfs/QmRZzu99ZiYsFhdKbdDjenih15yKNYXuap42aRDq9XH1J2"],"license":"Apache-2.0 OR MIT"}},"version":1},"id":48} \ No newline at end of file +{"abi":[{"type":"function","name":"getPDPConfig","inputs":[],"outputs":[{"name":"maxProvingPeriod","type":"uint64","internalType":"uint64"},{"name":"challengeWindow","type":"uint256","internalType":"uint256"},{"name":"challengesPerProof","type":"uint256","internalType":"uint256"},{"name":"initChallengeWindowStart","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"nextPDPChallengeWindowStart","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"getPDPConfig()":"ea0f9354","nextPDPChallengeWindowStart(uint256)":"11d41294"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"getPDPConfig\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"maxProvingPeriod\",\"type\":\"uint64\"},{\"internalType\":\"uint256\",\"name\":\"challengeWindow\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengesPerProof\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"initChallengeWindowStart\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"nextPDPChallengeWindowStart\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{\"getPDPConfig()\":{\"returns\":{\"challengeWindow\":\"Number of epochs for the challenge window\",\"challengesPerProof\":\"Number of challenges required per proof\",\"initChallengeWindowStart\":\"Initial challenge window start for new data sets assuming proving period starts now\",\"maxProvingPeriod\":\"Maximum number of epochs between proofs\"}},\"nextPDPChallengeWindowStart(uint256)\":{\"params\":{\"setId\":\"The ID of the data set\"},\"returns\":{\"_0\":\"The block number when the next challenge window starts\"}}},\"title\":\"IPDPProvingSchedule\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{\"getPDPConfig()\":{\"notice\":\"Returns PDP configuration values\"},\"nextPDPChallengeWindowStart(uint256)\":{\"notice\":\"Returns the start of the next challenge window for a data set\"}},\"notice\":\"Interface for PDP Service SLA specifications\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/IPDPProvingSchedule.sol\":\"IPDPProvingSchedule\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"src/IPDPProvingSchedule.sol\":{\"keccak256\":\"0x18f592eda642914eab092c28ae9527e20571a2a7191c82f475a432660c6a5417\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://142048503986dbb34905b03c99fed970d50dc0d088f2dc4274cc9e8c343ce83f\",\"dweb:/ipfs/QmbWYFT3ZuoefmSHmTrR64dinzxACfLKh9u3zHqRd1jETS\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[],"stateMutability":"view","type":"function","name":"getPDPConfig","outputs":[{"internalType":"uint64","name":"maxProvingPeriod","type":"uint64"},{"internalType":"uint256","name":"challengeWindow","type":"uint256"},{"internalType":"uint256","name":"challengesPerProof","type":"uint256"},{"internalType":"uint256","name":"initChallengeWindowStart","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"nextPDPChallengeWindowStart","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]}],"devdoc":{"kind":"dev","methods":{"getPDPConfig()":{"returns":{"challengeWindow":"Number of epochs for the challenge window","challengesPerProof":"Number of challenges required per proof","initChallengeWindowStart":"Initial challenge window start for new data sets assuming proving period starts now","maxProvingPeriod":"Maximum number of epochs between proofs"}},"nextPDPChallengeWindowStart(uint256)":{"params":{"setId":"The ID of the data set"},"returns":{"_0":"The block number when the next challenge window starts"}}},"version":1},"userdoc":{"kind":"user","methods":{"getPDPConfig()":{"notice":"Returns PDP configuration values"},"nextPDPChallengeWindowStart(uint256)":{"notice":"Returns the start of the next challenge window for a data set"}},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/IPDPProvingSchedule.sol":"IPDPProvingSchedule"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"src/IPDPProvingSchedule.sol":{"keccak256":"0x18f592eda642914eab092c28ae9527e20571a2a7191c82f475a432660c6a5417","urls":["bzz-raw://142048503986dbb34905b03c99fed970d50dc0d088f2dc4274cc9e8c343ce83f","dweb:/ipfs/QmbWYFT3ZuoefmSHmTrR64dinzxACfLKh9u3zHqRd1jETS"],"license":"Apache-2.0 OR MIT"}},"version":1},"id":48} \ No newline at end of file diff --git a/pdp/contract/PDPVerifier.abi b/pdp/contract/PDPVerifier.abi index 8ef493762..069b9a0bb 100644 --- a/pdp/contract/PDPVerifier.abi +++ b/pdp/contract/PDPVerifier.abi @@ -231,7 +231,7 @@ "internalType": "uint256" } ], - "stateMutability": "nonpayable" + "stateMutability": "view" }, { "type": "function", @@ -547,7 +547,7 @@ "internalType": "int32" } ], - "stateMutability": "nonpayable" + "stateMutability": "view" }, { "type": "function", @@ -1128,19 +1128,6 @@ ], "anonymous": false }, - { - "type": "event", - "name": "PriceOracleFailure", - "inputs": [ - { - "name": "reason", - "type": "bytes", - "indexed": false, - "internalType": "bytes" - } - ], - "anonymous": false - }, { "type": "event", "name": "ProofFeePaid", @@ -1306,4 +1293,4 @@ } ] } -] \ No newline at end of file +] diff --git a/pdp/contract/PDPVerifier.json b/pdp/contract/PDPVerifier.json index 0700ba02d..049cf67fa 100644 --- a/pdp/contract/PDPVerifier.json +++ b/pdp/contract/PDPVerifier.json @@ -1 +1 @@ -{"abi":[{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"newImplementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"},{"name":"pieceCids","type":"tuple[]","indexed":false,"internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"PriceOracleFailure","inputs":[{"name":"errorData","type":"bytes","indexed":false,"internalType":"bytes"}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"addPieces(uint256,(bytes)[],bytes)":"306fc8be","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getScheduledRemovals(uint256)":"6fa44692","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"indexed\":false,\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceCids\",\"type\":\"tuple[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"bytes\",\"name\":\"errorData\",\"type\":\"bytes\"}],\"name\":\"PriceOracleFailure\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"title\":\"IPDPVerifier\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"notice\":\"Main interface for the PDPVerifier contract\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/interfaces/IPDPVerifier.sol\":\"IPDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts/contracts/utils/Panic.sol\":{\"keccak256\":\"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a\",\"dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG\"]},\"lib/openzeppelin-contracts/contracts/utils/Strings.sol\":{\"keccak256\":\"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57\",\"dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn\"]},\"lib/openzeppelin-contracts/contracts/utils/math/Math.sol\":{\"keccak256\":\"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167\",\"dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol\":{\"keccak256\":\"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8\",\"dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol\":{\"keccak256\":\"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03\",\"dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04\",\"dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d\",\"dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0xc14a7a75b2b3d3be9db007a2314da8b71aa95c44114875dec69a2cdc5a89cdc4\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b037775b1d03dbccbf7eb71e8c8325c9772f1e006a003a186b6b0ebac08355d6\",\"dweb:/ipfs/QmeDeGPaoJ3RfNCFqrTirHZeb8NnExudNpS1kGk2VqQ7vM\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f\",\"dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t\"]},\"src/interfaces/IPDPVerifier.sol\":{\"keccak256\":\"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3\",\"dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"newImplementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false},{"internalType":"struct Cids.Cid[]","name":"pieceCids","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}],"indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"bytes","name":"errorData","type":"bytes","indexed":false}],"type":"event","name":"PriceOracleFailure","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct Cids.Cid[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"bytes","name":"","type":"bytes"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/interfaces/IPDPVerifier.sol":"IPDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts/contracts/utils/Panic.sol":{"keccak256":"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a","urls":["bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a","dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Strings.sol":{"keccak256":"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2","urls":["bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57","dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/Math.sol":{"keccak256":"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f","urls":["bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167","dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol":{"keccak256":"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54","urls":["bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8","dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol":{"keccak256":"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3","urls":["bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03","dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ"],"license":"MIT"},"src/BitOps.sol":{"keccak256":"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc","urls":["bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04","dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9","urls":["bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d","dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j"],"license":"Apache-2.0 OR MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0xc14a7a75b2b3d3be9db007a2314da8b71aa95c44114875dec69a2cdc5a89cdc4","urls":["bzz-raw://b037775b1d03dbccbf7eb71e8c8325c9772f1e006a003a186b6b0ebac08355d6","dweb:/ipfs/QmeDeGPaoJ3RfNCFqrTirHZeb8NnExudNpS1kGk2VqQ7vM"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846","urls":["bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f","dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t"],"license":"MIT"},"src/interfaces/IPDPVerifier.sol":{"keccak256":"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437","urls":["bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3","dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc"],"license":"MIT"}},"version":1},"id":53} \ No newline at end of file +{"abi":[{"type":"function","name":"addPieces","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceData","type":"tuple[]","internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"nonpayable"},{"type":"function","name":"claimDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"createDataSet","inputs":[{"name":"listenerAddr","type":"address","internalType":"address"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"payable"},{"type":"function","name":"dataSetLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"deleteDataSet","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"findPieceIds","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"leafIndexs","type":"uint256[]","internalType":"uint256[]"}],"outputs":[{"name":"","type":"tuple[]","internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"stateMutability":"view"},{"type":"function","name":"getChallengeFinality","inputs":[],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getChallengeRange","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLastProvenEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getDataSetListener","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"address","internalType":"address"},{"name":"","type":"address","internalType":"address"}],"stateMutability":"view"},{"type":"function","name":"getNextChallengeEpoch","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getNextDataSetId","inputs":[],"outputs":[{"name":"","type":"uint64","internalType":"uint64"}],"stateMutability":"view"},{"type":"function","name":"getNextPieceId","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getPieceCid","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bytes","internalType":"bytes"}],"stateMutability":"view"},{"type":"function","name":"getPieceLeafCount","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256","internalType":"uint256"}],"stateMutability":"view"},{"type":"function","name":"getScheduledRemovals","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"uint256[]","internalType":"uint256[]"}],"stateMutability":"view"},{"type":"function","name":"nextProvingPeriod","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","internalType":"uint256"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"pieceChallengable","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"pieceLive","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceId","type":"uint256","internalType":"uint256"}],"outputs":[{"name":"","type":"bool","internalType":"bool"}],"stateMutability":"view"},{"type":"function","name":"proposeDataSetStorageProvider","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"newStorageProvider","type":"address","internalType":"address"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"function","name":"provePossession","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"proofs","type":"tuple[]","internalType":"struct IPDPTypes.Proof[]","components":[{"name":"leaf","type":"bytes32","internalType":"bytes32"},{"name":"proof","type":"bytes32[]","internalType":"bytes32[]"}]}],"outputs":[],"stateMutability":"payable"},{"type":"function","name":"schedulePieceDeletions","inputs":[{"name":"setId","type":"uint256","internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","internalType":"uint256[]"},{"name":"extraData","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"ContractUpgraded","inputs":[{"name":"version","type":"string","indexed":false,"internalType":"string"},{"name":"newImplementation","type":"address","indexed":false,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetCreated","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"storageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false},{"type":"event","name":"DataSetDeleted","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"deletedLeafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"DataSetEmpty","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"NextProvingPeriod","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challengeEpoch","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"leafCount","type":"uint256","indexed":false,"internalType":"uint256"}],"anonymous":false},{"type":"event","name":"PiecesAdded","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"},{"name":"pieceCids","type":"tuple[]","indexed":false,"internalType":"struct Cids.Cid[]","components":[{"name":"data","type":"bytes","internalType":"bytes"}]}],"anonymous":false},{"type":"event","name":"PiecesRemoved","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"pieceIds","type":"uint256[]","indexed":false,"internalType":"uint256[]"}],"anonymous":false},{"type":"event","name":"PossessionProven","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"challenges","type":"tuple[]","indexed":false,"internalType":"struct IPDPTypes.PieceIdAndOffset[]","components":[{"name":"pieceId","type":"uint256","internalType":"uint256"},{"name":"offset","type":"uint256","internalType":"uint256"}]}],"anonymous":false},{"type":"event","name":"ProofFeePaid","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"fee","type":"uint256","indexed":false,"internalType":"uint256"},{"name":"price","type":"uint64","indexed":false,"internalType":"uint64"},{"name":"expo","type":"int32","indexed":false,"internalType":"int32"}],"anonymous":false},{"type":"event","name":"StorageProviderChanged","inputs":[{"name":"setId","type":"uint256","indexed":true,"internalType":"uint256"},{"name":"oldStorageProvider","type":"address","indexed":true,"internalType":"address"},{"name":"newStorageProvider","type":"address","indexed":true,"internalType":"address"}],"anonymous":false}],"bytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"deployedBytecode":{"object":"0x","sourceMap":"","linkReferences":{}},"methodIdentifiers":{"addPieces(uint256,(bytes)[],bytes)":"306fc8be","claimDataSetStorageProvider(uint256,bytes)":"df0f3248","createDataSet(address,bytes)":"bbae41cb","dataSetLive(uint256)":"ca759f27","deleteDataSet(uint256,bytes)":"7a1e2990","findPieceIds(uint256,uint256[])":"349c9179","getChallengeFinality()":"f83758fe","getChallengeRange(uint256)":"89208ba9","getDataSetLastProvenEpoch(uint256)":"04595c1a","getDataSetLeafCount(uint256)":"a531998c","getDataSetListener(uint256)":"2b3129bb","getDataSetStorageProvider(uint256)":"21b7cd1c","getNextChallengeEpoch(uint256)":"6ba4608f","getNextDataSetId()":"442cded3","getNextPieceId(uint256)":"1c5ae80f","getPieceCid(uint256,uint256)":"25bbbedf","getPieceLeafCount(uint256,uint256)":"0cd7b880","getScheduledRemovals(uint256)":"6fa44692","nextProvingPeriod(uint256,uint256,bytes)":"45c0b92d","pieceChallengable(uint256,uint256)":"dc635266","pieceLive(uint256,uint256)":"1a271225","proposeDataSetStorageProvider(uint256,address)":"43186080","provePossession(uint256,(bytes32,bytes32[])[])":"f58f952b","schedulePieceDeletions(uint256,uint256[],bytes)":"0c292024"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.23+commit.f704f362\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"version\",\"type\":\"string\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"newImplementation\",\"type\":\"address\"}],\"name\":\"ContractUpgraded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"storageProvider\",\"type\":\"address\"}],\"name\":\"DataSetCreated\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"deletedLeafCount\",\"type\":\"uint256\"}],\"name\":\"DataSetDeleted\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"DataSetEmpty\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"leafCount\",\"type\":\"uint256\"}],\"name\":\"NextProvingPeriod\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"indexed\":false,\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceCids\",\"type\":\"tuple[]\"}],\"name\":\"PiecesAdded\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"}],\"name\":\"PiecesRemoved\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"indexed\":false,\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"challenges\",\"type\":\"tuple[]\"}],\"name\":\"PossessionProven\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"fee\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"price\",\"type\":\"uint64\"},{\"indexed\":false,\"internalType\":\"int32\",\"name\":\"expo\",\"type\":\"int32\"}],\"name\":\"ProofFeePaid\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"oldStorageProvider\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"StorageProviderChanged\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes\",\"name\":\"data\",\"type\":\"bytes\"}],\"internalType\":\"struct Cids.Cid[]\",\"name\":\"pieceData\",\"type\":\"tuple[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"addPieces\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"claimDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"listenerAddr\",\"type\":\"address\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"createDataSet\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"dataSetLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"deleteDataSet\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"leafIndexs\",\"type\":\"uint256[]\"}],\"name\":\"findPieceIds\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"offset\",\"type\":\"uint256\"}],\"internalType\":\"struct IPDPTypes.PieceIdAndOffset[]\",\"name\":\"\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getChallengeFinality\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getChallengeRange\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLastProvenEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetListener\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getDataSetStorageProvider\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextChallengeEpoch\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"getNextDataSetId\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getNextPieceId\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceCid\",\"outputs\":[{\"internalType\":\"bytes\",\"name\":\"\",\"type\":\"bytes\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"getPieceLeafCount\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"}],\"name\":\"getScheduledRemovals\",\"outputs\":[{\"internalType\":\"uint256[]\",\"name\":\"\",\"type\":\"uint256[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"challengeEpoch\",\"type\":\"uint256\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"nextProvingPeriod\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceChallengable\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"pieceId\",\"type\":\"uint256\"}],\"name\":\"pieceLive\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"address\",\"name\":\"newStorageProvider\",\"type\":\"address\"}],\"name\":\"proposeDataSetStorageProvider\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"leaf\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32[]\",\"name\":\"proof\",\"type\":\"bytes32[]\"}],\"internalType\":\"struct IPDPTypes.Proof[]\",\"name\":\"proofs\",\"type\":\"tuple[]\"}],\"name\":\"provePossession\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"setId\",\"type\":\"uint256\"},{\"internalType\":\"uint256[]\",\"name\":\"pieceIds\",\"type\":\"uint256[]\"},{\"internalType\":\"bytes\",\"name\":\"extraData\",\"type\":\"bytes\"}],\"name\":\"schedulePieceDeletions\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"title\":\"IPDPVerifier\",\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"notice\":\"Main interface for the PDPVerifier contract\",\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/interfaces/IPDPVerifier.sol\":\"IPDPVerifier\"},\"evmVersion\":\"shanghai\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":1000000000},\"remappings\":[\":@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/\",\":@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/\",\":@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/\",\":erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/\",\":forge-std/=lib/forge-std/src/\",\":halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/\",\":openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/\",\":openzeppelin-contracts/=lib/openzeppelin-contracts/\",\":pyth-sdk-solidity/=lib/pyth-sdk-solidity/\"],\"viaIR\":true},\"sources\":{\"lib/openzeppelin-contracts/contracts/utils/Panic.sol\":{\"keccak256\":\"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a\",\"dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG\"]},\"lib/openzeppelin-contracts/contracts/utils/Strings.sol\":{\"keccak256\":\"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57\",\"dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn\"]},\"lib/openzeppelin-contracts/contracts/utils/math/Math.sol\":{\"keccak256\":\"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167\",\"dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol\":{\"keccak256\":\"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8\",\"dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy\"]},\"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol\":{\"keccak256\":\"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03\",\"dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ\"]},\"src/BitOps.sol\":{\"keccak256\":\"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04\",\"dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM\"]},\"src/Cids.sol\":{\"keccak256\":\"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9\",\"license\":\"Apache-2.0 OR MIT\",\"urls\":[\"bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d\",\"dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j\"]},\"src/interfaces/IPDPEvents.sol\":{\"keccak256\":\"0xbf68ec912762eea46e7121f579e1b9c8c04f2769a8535c012764db823450d356\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://02523a2fee16374e5c180e309cb91229cb822c8ac0beeeaae9501803d80e361a\",\"dweb:/ipfs/QmVgdVzjHJr77T7hRTqDcASpRHbfzH188743zNZmRSM5aQ\"]},\"src/interfaces/IPDPTypes.sol\":{\"keccak256\":\"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f\",\"dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t\"]},\"src/interfaces/IPDPVerifier.sol\":{\"keccak256\":\"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3\",\"dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.23+commit.f704f362"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"string","name":"version","type":"string","indexed":false},{"internalType":"address","name":"newImplementation","type":"address","indexed":false}],"type":"event","name":"ContractUpgraded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"storageProvider","type":"address","indexed":true}],"type":"event","name":"DataSetCreated","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"deletedLeafCount","type":"uint256","indexed":false}],"type":"event","name":"DataSetDeleted","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true}],"type":"event","name":"DataSetEmpty","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"challengeEpoch","type":"uint256","indexed":false},{"internalType":"uint256","name":"leafCount","type":"uint256","indexed":false}],"type":"event","name":"NextProvingPeriod","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false},{"internalType":"struct Cids.Cid[]","name":"pieceCids","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}],"indexed":false}],"type":"event","name":"PiecesAdded","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]","indexed":false}],"type":"event","name":"PiecesRemoved","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"challenges","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}],"indexed":false}],"type":"event","name":"PossessionProven","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"uint256","name":"fee","type":"uint256","indexed":false},{"internalType":"uint64","name":"price","type":"uint64","indexed":false},{"internalType":"int32","name":"expo","type":"int32","indexed":false}],"type":"event","name":"ProofFeePaid","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256","indexed":true},{"internalType":"address","name":"oldStorageProvider","type":"address","indexed":true},{"internalType":"address","name":"newStorageProvider","type":"address","indexed":true}],"type":"event","name":"StorageProviderChanged","anonymous":false},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct Cids.Cid[]","name":"pieceData","type":"tuple[]","components":[{"internalType":"bytes","name":"data","type":"bytes"}]},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"addPieces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"claimDataSetStorageProvider"},{"inputs":[{"internalType":"address","name":"listenerAddr","type":"address"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"payable","type":"function","name":"createDataSet","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"dataSetLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"deleteDataSet"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"leafIndexs","type":"uint256[]"}],"stateMutability":"view","type":"function","name":"findPieceIds","outputs":[{"internalType":"struct IPDPTypes.PieceIdAndOffset[]","name":"","type":"tuple[]","components":[{"internalType":"uint256","name":"pieceId","type":"uint256"},{"internalType":"uint256","name":"offset","type":"uint256"}]}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getChallengeFinality","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getChallengeRange","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLastProvenEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetListener","outputs":[{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getDataSetStorageProvider","outputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextChallengeEpoch","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[],"stateMutability":"view","type":"function","name":"getNextDataSetId","outputs":[{"internalType":"uint64","name":"","type":"uint64"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getNextPieceId","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceCid","outputs":[{"internalType":"bytes","name":"","type":"bytes"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getPieceLeafCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"}],"stateMutability":"view","type":"function","name":"getScheduledRemovals","outputs":[{"internalType":"uint256[]","name":"","type":"uint256[]"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"challengeEpoch","type":"uint256"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"nextProvingPeriod"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceChallengable","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256","name":"pieceId","type":"uint256"}],"stateMutability":"view","type":"function","name":"pieceLive","outputs":[{"internalType":"bool","name":"","type":"bool"}]},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"address","name":"newStorageProvider","type":"address"}],"stateMutability":"nonpayable","type":"function","name":"proposeDataSetStorageProvider"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"struct IPDPTypes.Proof[]","name":"proofs","type":"tuple[]","components":[{"internalType":"bytes32","name":"leaf","type":"bytes32"},{"internalType":"bytes32[]","name":"proof","type":"bytes32[]"}]}],"stateMutability":"payable","type":"function","name":"provePossession"},{"inputs":[{"internalType":"uint256","name":"setId","type":"uint256"},{"internalType":"uint256[]","name":"pieceIds","type":"uint256[]"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"schedulePieceDeletions"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":["@openzeppelin/contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/contracts/","@openzeppelin/contracts/=lib/openzeppelin-contracts/contracts/","@pythnetwork/pyth-sdk-solidity/=lib/pyth-sdk-solidity/","erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/","forge-std/=lib/forge-std/src/","halmos-cheatcodes/=lib/openzeppelin-contracts-upgradeable/lib/halmos-cheatcodes/src/","openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/","openzeppelin-contracts/=lib/openzeppelin-contracts/","pyth-sdk-solidity/=lib/pyth-sdk-solidity/"],"optimizer":{"enabled":true,"runs":1000000000},"metadata":{"bytecodeHash":"ipfs"},"compilationTarget":{"src/interfaces/IPDPVerifier.sol":"IPDPVerifier"},"evmVersion":"shanghai","libraries":{},"viaIR":true},"sources":{"lib/openzeppelin-contracts/contracts/utils/Panic.sol":{"keccak256":"0xf7fe324703a64fc51702311dc51562d5cb1497734f074e4f483bfb6717572d7a","urls":["bzz-raw://c6a5ff4f9fd8649b7ee20800b7fa387d3465bd77cf20c2d1068cd5c98e1ed57a","dweb:/ipfs/QmVSaVJf9FXFhdYEYeCEfjMVHrxDh5qL4CGkxdMWpQCrqG"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/Strings.sol":{"keccak256":"0x9e82c00fb176503860139c6bbf593b1a954ee7ff97ab2969656571382b9b58a2","urls":["bzz-raw://34b4eb157b44d4441315db65561ba7cf0fe909dc598c2cfd7080d203077d5b57","dweb:/ipfs/QmcNdvK3kDUAUr48urHxoeHd1TqVDya4YfZTM66i4goEJn"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/Math.sol":{"keccak256":"0x2c33f654cefbbe80a9b436b5792cfe8bda2e87f139f110073c99762558db252f","urls":["bzz-raw://dc7ebd80046a52f28978cf46a24ff3e4c8568264ab6bb138038951c75d576167","dweb:/ipfs/QmQQjXVr4CbDR3DXd8GHEqn3JSJYTnbBHMJp9tvc29yXrc"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SafeCast.sol":{"keccak256":"0x195533c86d0ef72bcc06456a4f66a9b941f38eb403739b00f21fd7c1abd1ae54","urls":["bzz-raw://b1d578337048cad08c1c03041cca5978eff5428aa130c781b271ad9e5566e1f8","dweb:/ipfs/QmPFKL2r9CBsMwmUqqdcFPfHZB2qcs9g1HDrPxzWSxomvy"],"license":"MIT"},"lib/openzeppelin-contracts/contracts/utils/math/SignedMath.sol":{"keccak256":"0xb1970fac7b64e6c09611e6691791e848d5e3fe410fa5899e7df2e0afd77a99e3","urls":["bzz-raw://db5fbb3dddd8b7047465b62575d96231ba8a2774d37fb4737fbf23340fabbb03","dweb:/ipfs/QmVUSvooZKEdEdap619tcJjTLcAuH6QBdZqAzWwnAXZAWJ"],"license":"MIT"},"src/BitOps.sol":{"keccak256":"0xe6447477342b60f948cb1785c7a723af7da96360887be0e23525604f960f69dc","urls":["bzz-raw://fe79487c0972995f4cf66c3c0c772777270ffd2faf3fdfcc2e5a974f953f6b04","dweb:/ipfs/QmPNrVcbPYr7kLQjczra7CicTvHijPjrvYBFCBEucKYppM"],"license":"Apache-2.0 OR MIT"},"src/Cids.sol":{"keccak256":"0x4a58ce512bbf6ba57a445b6887c8fb244ba1762d0ccc58e88eb25c13448074f9","urls":["bzz-raw://76a20f52425df5023d58eda12d8ddb7fc28839c75213e3275fb7eca4985bbd3d","dweb:/ipfs/QmcY1hevEnzUAN4nbNM2t2j8Hv284Ue9bB6fjQjktE1y6j"],"license":"Apache-2.0 OR MIT"},"src/interfaces/IPDPEvents.sol":{"keccak256":"0xbf68ec912762eea46e7121f579e1b9c8c04f2769a8535c012764db823450d356","urls":["bzz-raw://02523a2fee16374e5c180e309cb91229cb822c8ac0beeeaae9501803d80e361a","dweb:/ipfs/QmVgdVzjHJr77T7hRTqDcASpRHbfzH188743zNZmRSM5aQ"],"license":"MIT"},"src/interfaces/IPDPTypes.sol":{"keccak256":"0x1c5c9eb660a639c30b8b3cc09cc4d4c646935467440281b8c668365e237b6846","urls":["bzz-raw://91c470447e2922976856129e75670c9a4c64dd34c62442fa85a99a67a2add77f","dweb:/ipfs/QmNUXEUnhRRkfKzENo5dR1a1YWsytL7jMkGvxd913mat8t"],"license":"MIT"},"src/interfaces/IPDPVerifier.sol":{"keccak256":"0x9a315b3ac2d8700a75033f948a9f8234fd825ee970eda093dfac1ef920dc2437","urls":["bzz-raw://8de054dd020f346e35cdaff03e7944184e5272a78f80c801b3f879dba2f27dc3","dweb:/ipfs/QmdiVw2f9NmgdDukYfpWmnDPPJvHPikGdMWFzXJiFxRMcc"],"license":"MIT"}},"version":1},"id":54} \ No newline at end of file diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index f6aa46d96..1d2f36d1d 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -13,7 +13,7 @@ import ( const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" -const PDPTestNet = "0x549c257ddb5f9d6fbD195D10eE9e9B14a86D6DB6" +const PDPTestNet = "Change ME" type PDPContracts struct { PDPVerifier common.Address diff --git a/pdp/contract/pdp_proving_schedule.go b/pdp/contract/pdp_proving_schedule.go index 650cc5bdb..be0456667 100644 --- a/pdp/contract/pdp_proving_schedule.go +++ b/pdp/contract/pdp_proving_schedule.go @@ -31,7 +31,7 @@ var ( // IPDPProvingScheduleMetaData contains all meta data concerning the IPDPProvingSchedule contract. var IPDPProvingScheduleMetaData = &bind.MetaData{ - ABI: "[{\"type\":\"function\",\"name\":\"challengeWindow\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"pure\"},{\"type\":\"function\",\"name\":\"getChallengesPerProof\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"pure\"},{\"type\":\"function\",\"name\":\"getMaxProvingPeriod\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"pure\"},{\"type\":\"function\",\"name\":\"initChallengeWindowStart\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"pure\"},{\"type\":\"function\",\"name\":\"nextChallengeWindowStart\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"}]", + ABI: "[{\"type\":\"function\",\"name\":\"getPDPConfig\",\"inputs\":[],\"outputs\":[{\"name\":\"maxProvingPeriod\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"challengeWindow\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengesPerProof\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"initChallengeWindowStart\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"nextPDPChallengeWindowStart\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"}]", } // IPDPProvingScheduleABI is the input ABI used to generate the binding from. @@ -180,136 +180,67 @@ func (_IPDPProvingSchedule *IPDPProvingScheduleTransactorRaw) Transact(opts *bin return _IPDPProvingSchedule.Contract.contract.Transact(opts, method, params...) } -// ChallengeWindow is a free data retrieval call binding the contract method 0x861a1412. +// GetPDPConfig is a free data retrieval call binding the contract method 0xea0f9354. // -// Solidity: function challengeWindow() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) ChallengeWindow(opts *bind.CallOpts) (*big.Int, error) { +// Solidity: function getPDPConfig() view returns(uint64 maxProvingPeriod, uint256 challengeWindow, uint256 challengesPerProof, uint256 initChallengeWindowStart) +func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) GetPDPConfig(opts *bind.CallOpts) (struct { + MaxProvingPeriod uint64 + ChallengeWindow *big.Int + ChallengesPerProof *big.Int + InitChallengeWindowStart *big.Int +}, error) { var out []interface{} - err := _IPDPProvingSchedule.contract.Call(opts, &out, "challengeWindow") - + err := _IPDPProvingSchedule.contract.Call(opts, &out, "getPDPConfig") + + outstruct := new(struct { + MaxProvingPeriod uint64 + ChallengeWindow *big.Int + ChallengesPerProof *big.Int + InitChallengeWindowStart *big.Int + }) if err != nil { - return *new(*big.Int), err + return *outstruct, err } - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) - - return out0, err - -} + outstruct.MaxProvingPeriod = *abi.ConvertType(out[0], new(uint64)).(*uint64) + outstruct.ChallengeWindow = *abi.ConvertType(out[1], new(*big.Int)).(**big.Int) + outstruct.ChallengesPerProof = *abi.ConvertType(out[2], new(*big.Int)).(**big.Int) + outstruct.InitChallengeWindowStart = *abi.ConvertType(out[3], new(*big.Int)).(**big.Int) -// ChallengeWindow is a free data retrieval call binding the contract method 0x861a1412. -// -// Solidity: function challengeWindow() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleSession) ChallengeWindow() (*big.Int, error) { - return _IPDPProvingSchedule.Contract.ChallengeWindow(&_IPDPProvingSchedule.CallOpts) -} - -// ChallengeWindow is a free data retrieval call binding the contract method 0x861a1412. -// -// Solidity: function challengeWindow() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) ChallengeWindow() (*big.Int, error) { - return _IPDPProvingSchedule.Contract.ChallengeWindow(&_IPDPProvingSchedule.CallOpts) -} - -// GetChallengesPerProof is a free data retrieval call binding the contract method 0x47d3dfe7. -// -// Solidity: function getChallengesPerProof() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) GetChallengesPerProof(opts *bind.CallOpts) (uint64, error) { - var out []interface{} - err := _IPDPProvingSchedule.contract.Call(opts, &out, "getChallengesPerProof") - - if err != nil { - return *new(uint64), err - } - - out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) - - return out0, err - -} - -// GetChallengesPerProof is a free data retrieval call binding the contract method 0x47d3dfe7. -// -// Solidity: function getChallengesPerProof() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleSession) GetChallengesPerProof() (uint64, error) { - return _IPDPProvingSchedule.Contract.GetChallengesPerProof(&_IPDPProvingSchedule.CallOpts) -} - -// GetChallengesPerProof is a free data retrieval call binding the contract method 0x47d3dfe7. -// -// Solidity: function getChallengesPerProof() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) GetChallengesPerProof() (uint64, error) { - return _IPDPProvingSchedule.Contract.GetChallengesPerProof(&_IPDPProvingSchedule.CallOpts) -} - -// GetMaxProvingPeriod is a free data retrieval call binding the contract method 0xf2f12333. -// -// Solidity: function getMaxProvingPeriod() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) GetMaxProvingPeriod(opts *bind.CallOpts) (uint64, error) { - var out []interface{} - err := _IPDPProvingSchedule.contract.Call(opts, &out, "getMaxProvingPeriod") - - if err != nil { - return *new(uint64), err - } - - out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) - - return out0, err - -} - -// GetMaxProvingPeriod is a free data retrieval call binding the contract method 0xf2f12333. -// -// Solidity: function getMaxProvingPeriod() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleSession) GetMaxProvingPeriod() (uint64, error) { - return _IPDPProvingSchedule.Contract.GetMaxProvingPeriod(&_IPDPProvingSchedule.CallOpts) -} - -// GetMaxProvingPeriod is a free data retrieval call binding the contract method 0xf2f12333. -// -// Solidity: function getMaxProvingPeriod() pure returns(uint64) -func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) GetMaxProvingPeriod() (uint64, error) { - return _IPDPProvingSchedule.Contract.GetMaxProvingPeriod(&_IPDPProvingSchedule.CallOpts) -} - -// InitChallengeWindowStart is a free data retrieval call binding the contract method 0x21918cea. -// -// Solidity: function initChallengeWindowStart() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) InitChallengeWindowStart(opts *bind.CallOpts) (*big.Int, error) { - var out []interface{} - err := _IPDPProvingSchedule.contract.Call(opts, &out, "initChallengeWindowStart") - - if err != nil { - return *new(*big.Int), err - } - - out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) - - return out0, err + return *outstruct, err } -// InitChallengeWindowStart is a free data retrieval call binding the contract method 0x21918cea. +// GetPDPConfig is a free data retrieval call binding the contract method 0xea0f9354. // -// Solidity: function initChallengeWindowStart() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleSession) InitChallengeWindowStart() (*big.Int, error) { - return _IPDPProvingSchedule.Contract.InitChallengeWindowStart(&_IPDPProvingSchedule.CallOpts) +// Solidity: function getPDPConfig() view returns(uint64 maxProvingPeriod, uint256 challengeWindow, uint256 challengesPerProof, uint256 initChallengeWindowStart) +func (_IPDPProvingSchedule *IPDPProvingScheduleSession) GetPDPConfig() (struct { + MaxProvingPeriod uint64 + ChallengeWindow *big.Int + ChallengesPerProof *big.Int + InitChallengeWindowStart *big.Int +}, error) { + return _IPDPProvingSchedule.Contract.GetPDPConfig(&_IPDPProvingSchedule.CallOpts) } -// InitChallengeWindowStart is a free data retrieval call binding the contract method 0x21918cea. +// GetPDPConfig is a free data retrieval call binding the contract method 0xea0f9354. // -// Solidity: function initChallengeWindowStart() pure returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) InitChallengeWindowStart() (*big.Int, error) { - return _IPDPProvingSchedule.Contract.InitChallengeWindowStart(&_IPDPProvingSchedule.CallOpts) +// Solidity: function getPDPConfig() view returns(uint64 maxProvingPeriod, uint256 challengeWindow, uint256 challengesPerProof, uint256 initChallengeWindowStart) +func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) GetPDPConfig() (struct { + MaxProvingPeriod uint64 + ChallengeWindow *big.Int + ChallengesPerProof *big.Int + InitChallengeWindowStart *big.Int +}, error) { + return _IPDPProvingSchedule.Contract.GetPDPConfig(&_IPDPProvingSchedule.CallOpts) } -// NextChallengeWindowStart is a free data retrieval call binding the contract method 0x8bf96d28. +// NextPDPChallengeWindowStart is a free data retrieval call binding the contract method 0x11d41294. // -// Solidity: function nextChallengeWindowStart(uint256 setId) view returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) NextChallengeWindowStart(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { +// Solidity: function nextPDPChallengeWindowStart(uint256 setId) view returns(uint256) +func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) NextPDPChallengeWindowStart(opts *bind.CallOpts, setId *big.Int) (*big.Int, error) { var out []interface{} - err := _IPDPProvingSchedule.contract.Call(opts, &out, "nextChallengeWindowStart", setId) + err := _IPDPProvingSchedule.contract.Call(opts, &out, "nextPDPChallengeWindowStart", setId) if err != nil { return *new(*big.Int), err @@ -321,16 +252,16 @@ func (_IPDPProvingSchedule *IPDPProvingScheduleCaller) NextChallengeWindowStart( } -// NextChallengeWindowStart is a free data retrieval call binding the contract method 0x8bf96d28. +// NextPDPChallengeWindowStart is a free data retrieval call binding the contract method 0x11d41294. // -// Solidity: function nextChallengeWindowStart(uint256 setId) view returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleSession) NextChallengeWindowStart(setId *big.Int) (*big.Int, error) { - return _IPDPProvingSchedule.Contract.NextChallengeWindowStart(&_IPDPProvingSchedule.CallOpts, setId) +// Solidity: function nextPDPChallengeWindowStart(uint256 setId) view returns(uint256) +func (_IPDPProvingSchedule *IPDPProvingScheduleSession) NextPDPChallengeWindowStart(setId *big.Int) (*big.Int, error) { + return _IPDPProvingSchedule.Contract.NextPDPChallengeWindowStart(&_IPDPProvingSchedule.CallOpts, setId) } -// NextChallengeWindowStart is a free data retrieval call binding the contract method 0x8bf96d28. +// NextPDPChallengeWindowStart is a free data retrieval call binding the contract method 0x11d41294. // -// Solidity: function nextChallengeWindowStart(uint256 setId) view returns(uint256) -func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) NextChallengeWindowStart(setId *big.Int) (*big.Int, error) { - return _IPDPProvingSchedule.Contract.NextChallengeWindowStart(&_IPDPProvingSchedule.CallOpts, setId) +// Solidity: function nextPDPChallengeWindowStart(uint256 setId) view returns(uint256) +func (_IPDPProvingSchedule *IPDPProvingScheduleCallerSession) NextPDPChallengeWindowStart(setId *big.Int) (*big.Int, error) { + return _IPDPProvingSchedule.Contract.NextPDPChallengeWindowStart(&_IPDPProvingSchedule.CallOpts, setId) } diff --git a/pdp/contract/pdp_verifier.go b/pdp/contract/pdp_verifier.go index 287051c7a..e1957bdf0 100644 --- a/pdp/contract/pdp_verifier.go +++ b/pdp/contract/pdp_verifier.go @@ -48,7 +48,7 @@ type IPDPTypesProof struct { // PDPVerifierMetaData contains all meta data concerning the PDPVerifier contract. var PDPVerifierMetaData = &bind.MetaData{ - ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE_LOG2\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"activeCount\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"migrate\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"ContractUpgraded\",\"inputs\":[{\"name\":\"version\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":false,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"},{\"name\":\"pieceCids\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PriceOracleFailure\",\"inputs\":[{\"name\":\"reason\",\"type\":\"bytes\",\"indexed\":false,\"internalType\":\"bytes\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", + ABI: "[{\"type\":\"constructor\",\"inputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"BURN_ACTOR\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"EXTRA_DATA_MAX_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"FIL_USD_PRICE_FEED_ID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"LEAF_SIZE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_ENQUEUED_REMOVALS\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"MAX_PIECE_SIZE_LOG2\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_CHALLENGE_SCHEDULED\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"NO_PROVEN_EPOCH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"PYTH\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"contractIPyth\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"RANDOMNESS_PRECOMPILE\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"SECONDS_IN_DAY\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"UPGRADE_INTERFACE_VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"VERSION\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"string\",\"internalType\":\"string\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"addPieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceData\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"calculateProofFee\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"estimatedGasFee\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"claimDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"createDataSet\",\"inputs\":[{\"name\":\"listenerAddr\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"dataSetLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"deleteDataSet\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"findPieceIds\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"leafIndexs\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieceCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"activeCount\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getActivePieces\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"limit\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"pieces\",\"type\":\"tuple[]\",\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"rawSizes\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"hasMore\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeFinality\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getChallengeRange\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLastProvenEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetListener\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getFILUSDPrice\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"},{\"name\":\"\",\"type\":\"int32\",\"internalType\":\"int32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextChallengeEpoch\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextDataSetId\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"uint64\",\"internalType\":\"uint64\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getNextPieceId\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceCid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"tuple\",\"internalType\":\"structCids.Cid\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getPieceLeafCount\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getRandomness\",\"inputs\":[{\"name\":\"epoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"getScheduledRemovals\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"initialize\",\"inputs\":[{\"name\":\"_challengeFinality\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"migrate\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"nextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"owner\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceChallengable\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"pieceLive\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"}],\"outputs\":[{\"name\":\"\",\"type\":\"bool\",\"internalType\":\"bool\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"proposeDataSetStorageProvider\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"provePossession\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"proofs\",\"type\":\"tuple[]\",\"internalType\":\"structIPDPTypes.Proof[]\",\"components\":[{\"name\":\"leaf\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"},{\"name\":\"proof\",\"type\":\"bytes32[]\",\"internalType\":\"bytes32[]\"}]}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"function\",\"name\":\"proxiableUUID\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}],\"stateMutability\":\"view\"},{\"type\":\"function\",\"name\":\"renounceOwnership\",\"inputs\":[],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"schedulePieceDeletions\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"internalType\":\"uint256[]\"},{\"name\":\"extraData\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"transferOwnership\",\"inputs\":[{\"name\":\"newOwner\",\"type\":\"address\",\"internalType\":\"address\"}],\"outputs\":[],\"stateMutability\":\"nonpayable\"},{\"type\":\"function\",\"name\":\"upgradeToAndCall\",\"inputs\":[{\"name\":\"newImplementation\",\"type\":\"address\",\"internalType\":\"address\"},{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}],\"outputs\":[],\"stateMutability\":\"payable\"},{\"type\":\"event\",\"name\":\"ContractUpgraded\",\"inputs\":[{\"name\":\"version\",\"type\":\"string\",\"indexed\":false,\"internalType\":\"string\"},{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":false,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetCreated\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"storageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetDeleted\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"deletedLeafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"DataSetEmpty\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Initialized\",\"inputs\":[{\"name\":\"version\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"NextProvingPeriod\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challengeEpoch\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"leafCount\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"OwnershipTransferred\",\"inputs\":[{\"name\":\"previousOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newOwner\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesAdded\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"},{\"name\":\"pieceCids\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structCids.Cid[]\",\"components\":[{\"name\":\"data\",\"type\":\"bytes\",\"internalType\":\"bytes\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PiecesRemoved\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"pieceIds\",\"type\":\"uint256[]\",\"indexed\":false,\"internalType\":\"uint256[]\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"PossessionProven\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"challenges\",\"type\":\"tuple[]\",\"indexed\":false,\"internalType\":\"structIPDPTypes.PieceIdAndOffset[]\",\"components\":[{\"name\":\"pieceId\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"offset\",\"type\":\"uint256\",\"internalType\":\"uint256\"}]}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"ProofFeePaid\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"fee\",\"type\":\"uint256\",\"indexed\":false,\"internalType\":\"uint256\"},{\"name\":\"price\",\"type\":\"uint64\",\"indexed\":false,\"internalType\":\"uint64\"},{\"name\":\"expo\",\"type\":\"int32\",\"indexed\":false,\"internalType\":\"int32\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"StorageProviderChanged\",\"inputs\":[{\"name\":\"setId\",\"type\":\"uint256\",\"indexed\":true,\"internalType\":\"uint256\"},{\"name\":\"oldStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"},{\"name\":\"newStorageProvider\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"event\",\"name\":\"Upgraded\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"indexed\":true,\"internalType\":\"address\"}],\"anonymous\":false},{\"type\":\"error\",\"name\":\"AddressEmptyCode\",\"inputs\":[{\"name\":\"target\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967InvalidImplementation\",\"inputs\":[{\"name\":\"implementation\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"ERC1967NonPayable\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"FailedCall\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"IndexedError\",\"inputs\":[{\"name\":\"idx\",\"type\":\"uint256\",\"internalType\":\"uint256\"},{\"name\":\"msg\",\"type\":\"string\",\"internalType\":\"string\"}]},{\"type\":\"error\",\"name\":\"InvalidInitialization\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"NotInitializing\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"OwnableInvalidOwner\",\"inputs\":[{\"name\":\"owner\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"OwnableUnauthorizedAccount\",\"inputs\":[{\"name\":\"account\",\"type\":\"address\",\"internalType\":\"address\"}]},{\"type\":\"error\",\"name\":\"UUPSUnauthorizedCallContext\",\"inputs\":[]},{\"type\":\"error\",\"name\":\"UUPSUnsupportedProxiableUUID\",\"inputs\":[{\"name\":\"slot\",\"type\":\"bytes32\",\"internalType\":\"bytes32\"}]}]", } // PDPVerifierABI is the input ABI used to generate the binding from. @@ -600,6 +600,37 @@ func (_PDPVerifier *PDPVerifierCallerSession) VERSION() (string, error) { return _PDPVerifier.Contract.VERSION(&_PDPVerifier.CallOpts) } +// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) +func (_PDPVerifier *PDPVerifierCaller) CalculateProofFee(opts *bind.CallOpts, setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { + var out []interface{} + err := _PDPVerifier.contract.Call(opts, &out, "calculateProofFee", setId, estimatedGasFee) + + if err != nil { + return *new(*big.Int), err + } + + out0 := *abi.ConvertType(out[0], new(*big.Int)).(**big.Int) + + return out0, err + +} + +// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) +func (_PDPVerifier *PDPVerifierSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.CallOpts, setId, estimatedGasFee) +} + +// CalculateProofFee is a free data retrieval call binding the contract method 0x4903704a. +// +// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) view returns(uint256) +func (_PDPVerifier *PDPVerifierCallerSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*big.Int, error) { + return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.CallOpts, setId, estimatedGasFee) +} + // DataSetLive is a free data retrieval call binding the contract method 0xca759f27. // // Solidity: function dataSetLive(uint256 setId) view returns(bool) @@ -935,6 +966,38 @@ func (_PDPVerifier *PDPVerifierCallerSession) GetDataSetStorageProvider(setId *b return _PDPVerifier.Contract.GetDataSetStorageProvider(&_PDPVerifier.CallOpts, setId) } +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierCaller) GetFILUSDPrice(opts *bind.CallOpts) (uint64, int32, error) { + var out []interface{} + err := _PDPVerifier.contract.Call(opts, &out, "getFILUSDPrice") + + if err != nil { + return *new(uint64), *new(int32), err + } + + out0 := *abi.ConvertType(out[0], new(uint64)).(*uint64) + out1 := *abi.ConvertType(out[1], new(int32)).(*int32) + + return out0, out1, err + +} + +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (uint64, int32, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) +} + +// GetFILUSDPrice is a free data retrieval call binding the contract method 0x4fa27920. +// +// Solidity: function getFILUSDPrice() view returns(uint64, int32) +func (_PDPVerifier *PDPVerifierCallerSession) GetFILUSDPrice() (uint64, int32, error) { + return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.CallOpts) +} + // GetNextChallengeEpoch is a free data retrieval call binding the contract method 0x6ba4608f. // // Solidity: function getNextChallengeEpoch(uint256 setId) view returns(uint256) @@ -1297,27 +1360,6 @@ func (_PDPVerifier *PDPVerifierTransactorSession) AddPieces(setId *big.Int, piec return _PDPVerifier.Contract.AddPieces(&_PDPVerifier.TransactOpts, setId, pieceData, extraData) } -// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. -// -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactor) CalculateProofFee(opts *bind.TransactOpts, setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "calculateProofFee", setId, estimatedGasFee) -} - -// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. -// -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) -func (_PDPVerifier *PDPVerifierSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { - return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.TransactOpts, setId, estimatedGasFee) -} - -// CalculateProofFee is a paid mutator transaction binding the contract method 0x4903704a. -// -// Solidity: function calculateProofFee(uint256 setId, uint256 estimatedGasFee) returns(uint256) -func (_PDPVerifier *PDPVerifierTransactorSession) CalculateProofFee(setId *big.Int, estimatedGasFee *big.Int) (*types.Transaction, error) { - return _PDPVerifier.Contract.CalculateProofFee(&_PDPVerifier.TransactOpts, setId, estimatedGasFee) -} - // ClaimDataSetStorageProvider is a paid mutator transaction binding the contract method 0xdf0f3248. // // Solidity: function claimDataSetStorageProvider(uint256 setId, bytes extraData) returns() @@ -1381,27 +1423,6 @@ func (_PDPVerifier *PDPVerifierTransactorSession) DeleteDataSet(setId *big.Int, return _PDPVerifier.Contract.DeleteDataSet(&_PDPVerifier.TransactOpts, setId, extraData) } -// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() returns(uint64, int32) -func (_PDPVerifier *PDPVerifierTransactor) GetFILUSDPrice(opts *bind.TransactOpts) (*types.Transaction, error) { - return _PDPVerifier.contract.Transact(opts, "getFILUSDPrice") -} - -// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() returns(uint64, int32) -func (_PDPVerifier *PDPVerifierSession) GetFILUSDPrice() (*types.Transaction, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.TransactOpts) -} - -// GetFILUSDPrice is a paid mutator transaction binding the contract method 0x4fa27920. -// -// Solidity: function getFILUSDPrice() returns(uint64, int32) -func (_PDPVerifier *PDPVerifierTransactorSession) GetFILUSDPrice() (*types.Transaction, error) { - return _PDPVerifier.Contract.GetFILUSDPrice(&_PDPVerifier.TransactOpts) -} - // Initialize is a paid mutator transaction binding the contract method 0xfe4b84df. // // Solidity: function initialize(uint256 _challengeFinality) returns() @@ -3037,140 +3058,6 @@ func (_PDPVerifier *PDPVerifierFilterer) ParsePossessionProven(log types.Log) (* return event, nil } -// PDPVerifierPriceOracleFailureIterator is returned from FilterPriceOracleFailure and is used to iterate over the raw logs and unpacked data for PriceOracleFailure events raised by the PDPVerifier contract. -type PDPVerifierPriceOracleFailureIterator struct { - Event *PDPVerifierPriceOracleFailure // Event containing the contract specifics and raw log - - contract *bind.BoundContract // Generic contract to use for unpacking event data - event string // Event name to use for unpacking event data - - logs chan types.Log // Log channel receiving the found contract events - sub ethereum.Subscription // Subscription for errors, completion and termination - done bool // Whether the subscription completed delivering logs - fail error // Occurred error to stop iteration -} - -// Next advances the iterator to the subsequent event, returning whether there -// are any more events found. In case of a retrieval or parsing error, false is -// returned and Error() can be queried for the exact failure. -func (it *PDPVerifierPriceOracleFailureIterator) Next() bool { - // If the iterator failed, stop iterating - if it.fail != nil { - return false - } - // If the iterator completed, deliver directly whatever's available - if it.done { - select { - case log := <-it.logs: - it.Event = new(PDPVerifierPriceOracleFailure) - if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { - it.fail = err - return false - } - it.Event.Raw = log - return true - - default: - return false - } - } - // Iterator still in progress, wait for either a data or an error event - select { - case log := <-it.logs: - it.Event = new(PDPVerifierPriceOracleFailure) - if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil { - it.fail = err - return false - } - it.Event.Raw = log - return true - - case err := <-it.sub.Err(): - it.done = true - it.fail = err - return it.Next() - } -} - -// Error returns any retrieval or parsing error occurred during filtering. -func (it *PDPVerifierPriceOracleFailureIterator) Error() error { - return it.fail -} - -// Close terminates the iteration process, releasing any pending underlying -// resources. -func (it *PDPVerifierPriceOracleFailureIterator) Close() error { - it.sub.Unsubscribe() - return nil -} - -// PDPVerifierPriceOracleFailure represents a PriceOracleFailure event raised by the PDPVerifier contract. -type PDPVerifierPriceOracleFailure struct { - Reason []byte - Raw types.Log // Blockchain specific contextual infos -} - -// FilterPriceOracleFailure is a free log retrieval operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. -// -// Solidity: event PriceOracleFailure(bytes reason) -func (_PDPVerifier *PDPVerifierFilterer) FilterPriceOracleFailure(opts *bind.FilterOpts) (*PDPVerifierPriceOracleFailureIterator, error) { - - logs, sub, err := _PDPVerifier.contract.FilterLogs(opts, "PriceOracleFailure") - if err != nil { - return nil, err - } - return &PDPVerifierPriceOracleFailureIterator{contract: _PDPVerifier.contract, event: "PriceOracleFailure", logs: logs, sub: sub}, nil -} - -// WatchPriceOracleFailure is a free log subscription operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. -// -// Solidity: event PriceOracleFailure(bytes reason) -func (_PDPVerifier *PDPVerifierFilterer) WatchPriceOracleFailure(opts *bind.WatchOpts, sink chan<- *PDPVerifierPriceOracleFailure) (event.Subscription, error) { - - logs, sub, err := _PDPVerifier.contract.WatchLogs(opts, "PriceOracleFailure") - if err != nil { - return nil, err - } - return event.NewSubscription(func(quit <-chan struct{}) error { - defer sub.Unsubscribe() - for { - select { - case log := <-logs: - // New log arrived, parse the event and forward to the user - event := new(PDPVerifierPriceOracleFailure) - if err := _PDPVerifier.contract.UnpackLog(event, "PriceOracleFailure", log); err != nil { - return err - } - event.Raw = log - - select { - case sink <- event: - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - case err := <-sub.Err(): - return err - case <-quit: - return nil - } - } - }), nil -} - -// ParsePriceOracleFailure is a log parse operation binding the contract event 0x32e677043f93cd9edc909002276c5e55eb11c275a7b368ae7fe86687516eb120. -// -// Solidity: event PriceOracleFailure(bytes reason) -func (_PDPVerifier *PDPVerifierFilterer) ParsePriceOracleFailure(log types.Log) (*PDPVerifierPriceOracleFailure, error) { - event := new(PDPVerifierPriceOracleFailure) - if err := _PDPVerifier.contract.UnpackLog(event, "PriceOracleFailure", log); err != nil { - return nil, err - } - event.Raw = log - return event, nil -} - // PDPVerifierProofFeePaidIterator is returned from FilterProofFeePaid and is used to iterate over the raw logs and unpacked data for ProofFeePaid events raised by the PDPVerifier contract. type PDPVerifierProofFeePaidIterator struct { Event *PDPVerifierProofFeePaid // Event containing the contract specifics and raw log diff --git a/tasks/pdp/data_set_create_watch.go b/tasks/pdp/data_set_create_watch.go index d21f602c6..1a3f2b0c2 100644 --- a/tasks/pdp/data_set_create_watch.go +++ b/tasks/pdp/data_set_create_watch.go @@ -213,16 +213,10 @@ func getProvingPeriodChallengeWindow(ctx context.Context, ethClient *ethclient.C return 0, 0, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) } - period, err := schedule.GetMaxProvingPeriod(&bind.CallOpts{Context: ctx}) + config, err := schedule.GetPDPConfig(&bind.CallOpts{Context: ctx}) if err != nil { - return 0, 0, xerrors.Errorf("failed to get proving period: %w", err) + return 0, 0, xerrors.Errorf("failed to get pdp config: %w", err) } - // ChallengeWindow - challengeWindow, err := schedule.ChallengeWindow(&bind.CallOpts{Context: ctx}) - if err != nil { - return 0, 0, xerrors.Errorf("failed to get challenge window: %w", err) - } - - return period, challengeWindow.Uint64(), nil + return config.MaxProvingPeriod, config.ChallengeWindow.Uint64(), nil } diff --git a/tasks/pdp/task_init_pp.go b/tasks/pdp/task_init_pp.go index d85cb1ba8..1ab30d594 100644 --- a/tasks/pdp/task_init_pp.go +++ b/tasks/pdp/task_init_pp.go @@ -137,16 +137,15 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( return false, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) } - // ChallengeWindow - challengeWindow, err := provingSchedule.ChallengeWindow(&bind.CallOpts{Context: ctx}) + config, err := provingSchedule.GetPDPConfig(&bind.CallOpts{Context: ctx}) if err != nil { - return false, xerrors.Errorf("failed to get challenge window: %w", err) + return false, xerrors.Errorf("failed to get pdp config: %w", err) } - init_prove_at, err := provingSchedule.InitChallengeWindowStart(&bind.CallOpts{Context: ctx}) - if err != nil { - return false, xerrors.Errorf("failed to get next challenge window start: %w", err) - } + // ChallengeWindow + challengeWindow := config.ChallengeWindow + + init_prove_at := config.InitChallengeWindowStart init_prove_at = init_prove_at.Add(init_prove_at, challengeWindow.Div(challengeWindow, big.NewInt(2))) // Give a buffer of 1/2 challenge window epochs so that we are still within challenge window // Instantiate the PDPVerifier contract pdpContracts := contract.ContractAddresses() diff --git a/tasks/pdp/task_next_pp.go b/tasks/pdp/task_next_pp.go index e3f0d6c8a..c1458e970 100644 --- a/tasks/pdp/task_next_pp.go +++ b/tasks/pdp/task_next_pp.go @@ -123,7 +123,8 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() if err != nil { return false, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) } - next_prove_at, err := provingSchedule.NextChallengeWindowStart(nil, big.NewInt(dataSetID)) + + next_prove_at, err := provingSchedule.NextPDPChallengeWindowStart(nil, big.NewInt(dataSetID)) if err != nil { return false, xerrors.Errorf("failed to get next challenge window start: %w", err) } From 5d4ad52c0c58f94f4f5bf27995951542fc242878 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Thu, 4 Sep 2025 19:05:38 +0400 Subject: [PATCH 31/55] fix swagger definitions --- .../harmonydb/sql/20250505-market-mk20.sql | 59 +-- market/mk20/client/client.go | 418 +++++++++++------- market/mk20/client/http_client.go | 6 +- market/mk20/ddo_v1.go | 8 +- market/mk20/http/docs.go | 149 +++++-- market/mk20/http/http.go | 92 ++-- market/mk20/http/swagger.json | 149 +++++-- market/mk20/http/swagger.yaml | 95 ++-- market/mk20/mk20_utils.go | 14 +- market/mk20/pdp_v1.go | 6 +- market/mk20/types.go | 10 +- 11 files changed, 625 insertions(+), 381 deletions(-) diff --git a/harmony/harmonydb/sql/20250505-market-mk20.sql b/harmony/harmonydb/sql/20250505-market-mk20.sql index 80c3101af..a1398e1d5 100644 --- a/harmony/harmonydb/sql/20250505-market-mk20.sql +++ b/harmony/harmonydb/sql/20250505-market-mk20.sql @@ -53,44 +53,6 @@ ALTER TABLE ipni -- The order_number column must be completely sequential ALTER SEQUENCE ipni_order_number_seq CACHE 1; --- Add a column in ipni_head to reference a specific ipni row -ALTER TABLE ipni_head - ADD COLUMN head_order_number BIGINT; - --- Backfill head_order_number to the row you intend as the head. --- If "head" should point to the latest row with that ad_cid/provider: -WITH latest AS ( - SELECT h.provider, h.head, - MAX(i.order_number) AS order_number - FROM ipni_head h - JOIN ipni i - ON i.provider = h.provider - AND i.ad_cid = h.head - GROUP BY h.provider, h.head -) -UPDATE ipni_head h -SET head_order_number = l.order_number - FROM latest l -WHERE h.provider = l.provider AND h.head = l.head; - --- Make it NOT NULL once backfilled -ALTER TABLE ipni_head - ALTER COLUMN head_order_number SET NOT NULL; - --- Switch the FK to reference the unique parent key -ALTER TABLE ipni_head DROP CONSTRAINT ipni_head_head_fkey; - -ALTER TABLE ipni_head - ADD CONSTRAINT ipni_head_head_order_fkey - FOREIGN KEY (head_order_number) - REFERENCES ipni(order_number) - ON DELETE RESTRICT; - --- Now remove uniqueness on ad_cid (both enforcers). This allows us --- to chain add/delete/ad/delete for same piece -ALTER TABLE ipni DROP CONSTRAINT ipni_ad_cid_key; -DROP INDEX ipni_ad_cid; - -- This function is used to insert piece metadata and piece deal (piece indexing) -- This makes it easy to keep the logic of how table is updated and fast (in DB). CREATE OR REPLACE FUNCTION process_piece_deal( @@ -209,12 +171,6 @@ BEGIN; AND p.raw_size IS NOT NULL; COMMIT; - -CREATE TABLE ddo_contracts ( - address TEXT NOT NULL PRIMARY KEY, - abi TEXT NOT NULL -); - -- This is main MK20 Deal table. Rows are added per deal and some -- modification is allowed later CREATE TABLE market_mk20_deal ( @@ -230,6 +186,8 @@ CREATE TABLE market_mk20_deal ( retrieval_v1 JSONB NOT NULL DEFAULT 'null', pdp_v1 JSONB NOT NULL DEFAULT 'null' ); +COMMENT ON COLUMN market_mk20_deal.id IS 'This is ULID TEXT'; +COMMENT ON COLUMN market_mk20_deal.client IS 'Client must always be text as this can be a non Filecoin address like ed25519'; -- This is main pipeline table for PoRep processing of MK20 deals CREATE TABLE market_mk20_pipeline ( @@ -248,7 +206,7 @@ CREATE TABLE market_mk20_pipeline ( announce BOOLEAN NOT NULL, allocation_id BIGINT DEFAULT NULL, duration BIGINT NOT NULL, - piece_aggregation INT NOT NULL DEFAULT 0, + piece_aggregation INT NOT NULL DEFAULT 0, -- This is set when user sends a aggregated piece. It is also set as `deal_aggregation` when deal is aggregated on SP side. started BOOLEAN DEFAULT FALSE, @@ -276,6 +234,8 @@ CREATE TABLE market_mk20_pipeline ( PRIMARY KEY (id, aggr_index) ); +COMMENT ON COLUMN market_mk20_pipeline.piece_aggregation IS 'This is set when user sends a aggregated piece. It is also set as `deal_aggregation` when deal is aggregated on SP side.'; +COMMENT ON COLUMN market_mk20_pipeline.deal_aggregation IS 'This is set when user sends a deal with aggregated source. This value is passed to piece_aggregation when aggregation is finished and a single piece remains'; -- This table is used to hold MK20 deals waiting for PoRep pipeline -- to process. This allows disconnecting the need to immediately process @@ -720,13 +680,12 @@ BEGIN -- Insert the new ad into the ipni table with an automatically assigned order_number INSERT INTO ipni (ad_cid, context_id, metadata, is_rm, previous, provider, addresses, signature, entries, piece_cid_v2, piece_cid, piece_size) - VALUES (_ad_cid, _context_id, _metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size) RETURNING order_number INTO _new_order; + VALUES (_ad_cid, _context_id, _metadata, _is_rm, _previous, _provider, _addresses, _signature, _entries, _piece_cid_v2, _piece_cid, _piece_size); -- Update the ipni_head table to set the new ad as the head of the chain - INSERT INTO ipni_head (provider, head, head_order_number) - VALUES (_provider, _ad_cid, _new_order) - ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head, - head_order_number = EXCLUDED.head_order_number; + INSERT INTO ipni_head (provider, head) + VALUES (_provider, _ad_cid) + ON CONFLICT (provider) DO UPDATE SET head = EXCLUDED.head; END; $$ LANGUAGE plpgsql; diff --git a/market/mk20/client/client.go b/market/mk20/client/client.go index dc08af9d8..78417c333 100644 --- a/market/mk20/client/client.go +++ b/market/mk20/client/client.go @@ -1,27 +1,20 @@ package client import ( - "bufio" "bytes" "context" - "fmt" + "crypto/rand" "io" - "net/http" - "net/url" - "os" - "strings" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/wallet" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "github.com/mitchellh/go-homedir" "github.com/oklog/ulid" "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v16/verifreg" - "github.com/filecoin-project/curio/market/mk20" + "github.com/filecoin-project/go-address" ) var log = logging.Logger("mk20-client") @@ -30,178 +23,270 @@ type Client struct { http *HTTPClient } -func NewClient(baseURL, auth string) *Client { - hclient := New(baseURL, Option(WithAuthString(auth))) +func NewClient(baseURL string, client address.Address, wallet *wallet.LocalWallet) *Client { + s := NewAuth(client, wallet) + hclient := NewHTTPClient(baseURL, HourlyCurioAuthHeader(s)) return &Client{ http: hclient, } } -func (c *Client) Deal(ctx context.Context, maddr, wallet address.Address, pieceCid cid.Cid, http_url, aggregateFile, contract_address, contract_method string, headers http.Header, put, index, announce, pdp bool, duration, allocation, proofSet int64) error { - var d mk20.DataSource +func (c *Client) CreateDataSet(ctx context.Context, client, recordKeeper string, extraData []byte) (ulid.ULID, error) { + id, err := ulid.New(ulid.Now(), rand.Reader) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create ULID: %w", err) + } - if aggregateFile != "" { - d = mk20.DataSource{ - PieceCID: pieceCid, - Format: mk20.PieceDataFormat{ - Aggregate: &mk20.FormatAggregate{ - Type: mk20.AggregateTypeV1, - }, + deal := &mk20.Deal{ + Identifier: id, + Client: client, + Products: mk20.Products{ + PDPV1: &mk20.PDPV1{ + CreateDataSet: true, + RecordKeeper: recordKeeper, + ExtraData: extraData, }, - } + }, + } - var pieces []mk20.DataSource + rerr := c.http.Store(ctx, deal) + if rerr.Error != nil { + return ulid.ULID{}, rerr.Error + } + if rerr.Status != 200 { + return ulid.ULID{}, rerr.HError() + } + return id, nil +} - log.Debugw("using aggregate data source", "aggregate", aggregateFile) - // Read file line by line - loc, err := homedir.Expand(aggregateFile) - if err != nil { - return err - } - file, err := os.Open(loc) - if err != nil { - return err - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Split(line, "\t") - if len(parts) != 2 { - return fmt.Errorf("invalid line format. Expected pieceCidV2, url at %s", line) - } - if parts[0] == "" || parts[1] == "" { - return fmt.Errorf("empty column value in the input file at %s", line) - } +func (c *Client) RemoveDataSet(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64) (ulid.ULID, error) { + if dataSetID == nil { + return ulid.ULID{}, nil + } - pieceCid, err := cid.Parse(parts[0]) - if err != nil { - return fmt.Errorf("failed to parse CID: %w", err) - } + id, err := ulid.New(ulid.Now(), rand.Reader) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create ULID: %w", err) + } - url, err := url.Parse(parts[1]) - if err != nil { - return fmt.Errorf("failed to parse url: %w", err) - } + deal := &mk20.Deal{ + Identifier: id, + Client: client, + Products: mk20.Products{ + PDPV1: &mk20.PDPV1{ + DeleteDataSet: true, + DataSetID: dataSetID, + RecordKeeper: recordKeeper, + ExtraData: extraData, + }, + }, + } - pieces = append(pieces, mk20.DataSource{ - PieceCID: pieceCid, - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceHTTP: &mk20.DataSourceHTTP{ - URLs: []mk20.HttpUrl{ - { - URL: url.String(), - Priority: 0, - Fallback: true, - }, - }, - }, - }) - - if err := scanner.Err(); err != nil { - return err - } + rerr := c.http.Store(ctx, deal) + if rerr.Error != nil { + return ulid.ULID{}, rerr.Error + } + if rerr.Status != 200 { + return ulid.ULID{}, rerr.HError() + } + return id, nil +} + +func (c *Client) addPiece(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, dataSource *mk20.DataSource, ret *mk20.RetrievalV1) (ulid.ULID, error) { + if dataSetID == nil { + return ulid.ULID{}, nil + } + + id, err := ulid.New(ulid.Now(), rand.Reader) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create ULID: %w", err) + } + + deal := &mk20.Deal{ + Identifier: id, + Client: client, + Data: dataSource, + Products: mk20.Products{ + PDPV1: &mk20.PDPV1{ + AddPiece: true, + DataSetID: dataSetID, + RecordKeeper: recordKeeper, + ExtraData: extraData, + }, + RetrievalV1: ret, + }, + } + + rerr := c.http.Store(ctx, deal) + if rerr.Error != nil { + return ulid.ULID{}, rerr.Error + } + if rerr.Status != 200 { + return ulid.ULID{}, rerr.HError() + } + return id, nil +} + +func (c *Client) RemovePiece(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, pieceIDs []uint64) (ulid.ULID, error) { + if dataSetID == nil { + return ulid.ULID{}, xerrors.Errorf("dataSetID is required") + } + + if len(pieceIDs) == 0 { + return ulid.ULID{}, xerrors.Errorf("at least one pieceID is required") + } + + id, err := ulid.New(ulid.Now(), rand.Reader) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create ULID: %w", err) + } + + deal := &mk20.Deal{ + Identifier: id, + Client: client, + Products: mk20.Products{ + PDPV1: &mk20.PDPV1{ + DeletePiece: true, + DataSetID: dataSetID, + RecordKeeper: recordKeeper, + ExtraData: extraData, + PieceIDs: pieceIDs, + }, + }, + } + + rerr := c.http.Store(ctx, deal) + if rerr.Error != nil { + return ulid.ULID{}, rerr.Error + } + if rerr.Status != 200 { + return ulid.ULID{}, rerr.HError() + } + return id, nil +} + +func (c *Client) CreateDataSource(pieceCID cid.Cid, car, raw, aggregate, index, withCDN bool, aggregateType mk20.AggregateType, sub []mk20.DataSource) (*mk20.Deal, error) { + if car && raw && aggregate || car && raw || car && aggregate || raw && aggregate { + return nil, xerrors.Errorf("only one data format is supported") + } + + if !car && (index || withCDN) { + return nil, xerrors.Errorf("only car data format supports IPFS style CDN retrievals") + } + + err := mk20.ValidatePieceCID(pieceCID) + if err != nil { + return nil, err + } + + dataSource := &mk20.DataSource{ + PieceCID: pieceCID, + } + + if car { + dataSource.Format.Car = &mk20.FormatCar{} + } + + if raw { + dataSource.Format.Raw = &mk20.FormatBytes{} + } + + if aggregate { + if len(sub) <= 1 { + return nil, xerrors.Errorf("must provide at least two sub data source") } - d.SourceAggregate = &mk20.DataSourceAggregate{ - Pieces: pieces, + + if aggregateType == mk20.AggregateTypeNone { + return nil, xerrors.Errorf("must provide valid aggregateType") } - } else { - if http_url == "" { - if put { - d = mk20.DataSource{ - PieceCID: pieceCid, - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceHttpPut: &mk20.DataSourceHttpPut{}, - } - } else { - d = mk20.DataSource{ - PieceCID: pieceCid, - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceOffline: &mk20.DataSourceOffline{}, - } - } - } else { - url, err := url.Parse(http_url) - if err != nil { - return xerrors.Errorf("parsing http url: %w", err) - } - d = mk20.DataSource{ - PieceCID: pieceCid, - Format: mk20.PieceDataFormat{ - Car: &mk20.FormatCar{}, - }, - SourceHTTP: &mk20.DataSourceHTTP{ - URLs: []mk20.HttpUrl{ - { - URL: url.String(), - Headers: headers, - Priority: 0, - Fallback: true, - }, - }, - }, - } + + dataSource.Format.Aggregate = &mk20.FormatAggregate{ + Type: aggregateType, + Sub: sub, } } - p := mk20.Products{ - DDOV1: &mk20.DDOV1{ - Provider: maddr, - PieceManager: wallet, - Duration: abi.ChainEpoch(duration), - ContractAddress: contract_address, - ContractVerifyMethod: contract_method, - ContractVerifyMethodParams: []byte("test bytes"), - }, - RetrievalV1: &mk20.RetrievalV1{ - Indexing: index, - AnnouncePayload: announce, + ret := &mk20.Deal{ + Data: dataSource, + Products: mk20.Products{ + RetrievalV1: &mk20.RetrievalV1{ + Indexing: index, + AnnouncePiece: true, + AnnouncePayload: withCDN, + }, }, } - if pdp { - ps := uint64(proofSet) - p.PDPV1 = &mk20.PDPV1{ - AddPiece: true, - DataSetID: &ps, - ExtraData: []byte("test bytes"), // TODO: Fix this - } + return ret, nil +} + +func (c *Client) AddPieceWithHTTP(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, pieceCID cid.Cid, car, raw, index, withCDN bool, aggregateType mk20.AggregateType, sub []mk20.DataSource, urls []mk20.HttpUrl) (ulid.ULID, error) { + var aggregate bool + + if aggregateType == mk20.AggregateTypeV1 { + aggregate = true + } + + d, err := c.CreateDataSource(pieceCID, car, raw, aggregate, index, withCDN, aggregateType, sub) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create data source: %w", err) } - if allocation != 0 { - alloc := verifreg.AllocationId(allocation) - p.DDOV1.AllocationId = &alloc + d.Data.SourceHTTP = &mk20.DataSourceHTTP{ + URLs: urls, } - id, err := mk20.NewULID() + return c.addPiece(ctx, client, recordKeeper, extraData, dataSetID, d.Data, d.Products.RetrievalV1) +} + +func (c *Client) AddPieceWithAggregate(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, pieceCID cid.Cid, index, withCDN bool, aggregateType mk20.AggregateType, sub []mk20.DataSource) (ulid.ULID, error) { + d, err := c.CreateDataSource(pieceCID, false, false, true, index, withCDN, aggregateType, sub) if err != nil { - return err + return ulid.ULID{}, xerrors.Errorf("failed to create data source: %w", err) } - log.Debugw("generated deal id", "id", id) - deal := mk20.Deal{ - Identifier: id, - Client: wallet.String(), - Data: &d, - Products: p, + d.Data.SourceAggregate = &mk20.DataSourceAggregate{ + Pieces: sub, } - log.Debugw("deal", "deal", deal) + d.Data.Format.Aggregate.Sub = nil - rerr := c.http.Store(ctx, &deal) - if rerr.Error != nil { - return rerr.Error + return c.addPiece(ctx, client, recordKeeper, extraData, dataSetID, d.Data, d.Products.RetrievalV1) +} + +func (c *Client) AddPieceWithPut(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, pieceCID cid.Cid, car, raw, index, withCDN bool, aggregateType mk20.AggregateType, sub []mk20.DataSource) (ulid.ULID, error) { + var aggregate bool + + if aggregateType == mk20.AggregateTypeV1 { + aggregate = true } - if rerr.Status != 200 { - return rerr.HError() + + d, err := c.CreateDataSource(pieceCID, car, raw, aggregate, index, withCDN, aggregateType, sub) + if err != nil { + return ulid.ULID{}, xerrors.Errorf("failed to create data source: %w", err) } - return nil + + d.Data.SourceHttpPut = &mk20.DataSourceHttpPut{} + + return c.addPiece(ctx, client, recordKeeper, extraData, dataSetID, d.Data, d.Products.RetrievalV1) +} + +func (c *Client) AddPieceWithPutStreaming(ctx context.Context, client, recordKeeper string, extraData []byte, dataSetID *uint64, car, raw, aggregate, index, withCDN bool) (ulid.ULID, error) { + if car && raw && aggregate || car && raw || car && aggregate || raw && aggregate { + return ulid.ULID{}, xerrors.Errorf("only one data format is supported") + } + + if !car && (index || withCDN) { + return ulid.ULID{}, xerrors.Errorf("only car data format supports IPFS style CDN retrievals") + } + + ret := &mk20.RetrievalV1{ + Indexing: index, + AnnouncePiece: true, + AnnouncePayload: withCDN, + } + + return c.addPiece(ctx, client, recordKeeper, extraData, dataSetID, nil, ret) } func (c *Client) DealStatus(ctx context.Context, dealID string) (*mk20.DealProductStatusResponse, error) { @@ -415,3 +500,34 @@ func KeyFromClientAddress(clientAddress address.Address) (key string) { return "" } } + +type ClientAuth struct { + client address.Address + wallet *wallet.LocalWallet +} + +func (c *ClientAuth) Sign(digest []byte) ([]byte, error) { + sign, err := c.wallet.WalletSign(context.Background(), c.client, digest, lapi.MsgMeta{Type: lapi.MTDealProposal}) + if err != nil { + return nil, err + } + + return sign.MarshalBinary() +} + +func (c *ClientAuth) PublicKeyBytes() []byte { + return c.client.Bytes() +} + +func (c *ClientAuth) Type() string { + return KeyFromClientAddress(c.client) +} + +var _ Signer = &ClientAuth{} + +func NewAuth(client address.Address, wallet *wallet.LocalWallet) Signer { + return &ClientAuth{ + client: client, + wallet: wallet, + } +} diff --git a/market/mk20/client/http_client.go b/market/mk20/client/http_client.go index b44cfbb56..47bf81a05 100644 --- a/market/mk20/client/http_client.go +++ b/market/mk20/client/http_client.go @@ -27,8 +27,8 @@ type HTTPClient struct { AuthHeaderString string } -// New returns a HTTPClient with sane defaults. -func New(baseURL string, opts ...Option) *HTTPClient { +// NewHTTPClient returns a HTTPClient with sane defaults. +func NewHTTPClient(baseURL string, opts ...Option) *HTTPClient { c := &HTTPClient{ BaseURL: baseURL + MarketPath, HTTP: &http.Client{Timeout: 60 * time.Second}, @@ -104,7 +104,7 @@ func (c *HTTPClient) do(ctx context.Context, method, p string, body io.Reader, v return &Error{Status: resp.StatusCode, Error: err} } } - return nil + return &Error{Status: resp.StatusCode} } // Error wraps nonā€‘2xx responses. diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index 3993214cd..9a1f1c059 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -40,8 +40,8 @@ type DDOV1 struct { // It must be at least 518400 Duration abi.ChainEpoch `json:"duration"` - // AllocationId represents an aggregated allocation identifier for the deal. - AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty"` + // AllocationId represents an allocation identifier for the deal. + AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty" swaggertype:"integer" format:"uint64" example:"1"` // ContractAddress specifies the address of the contract governing the deal ContractAddress string `json:"contract_address"` @@ -50,13 +50,13 @@ type DDOV1 struct { ContractVerifyMethod string `json:"contract_verify_method"` // ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - ContractVerifyMethodParams []byte `json:"contract_verify_method_params,omitempty"` + ContractVerifyMethodParams []byte `json:"contract_verify_method_params,omitempty" swaggertype:"string" format:"byte"` // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated NotificationAddress string `json:"notification_address"` // NotificationPayload holds the notification data typically in a serialized byte array format. - NotificationPayload []byte `json:"notification_payload,omitempty"` + NotificationPayload []byte `json:"notification_payload,omitempty" swaggertype:"string" format:"byte"` } func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 45b30198a..cfb77ede3 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -35,6 +35,27 @@ const docTemplate = `{ } } }, + "/info/": { + "get": { + "description": "- OpenAPI spec UI for the Market 2.0 APIs", + "summary": "OpenAPI Spec UI", + "responses": {} + } + }, + "/info/swagger.json": { + "get": { + "description": "- OpenAPI spec for the Market 2.0 APIs in JSON format", + "summary": "OpenAPI Spec JSON", + "responses": {} + } + }, + "/info/swagger.yaml": { + "get": { + "description": "- OpenAPI spec for the Market 2.0 APIs in YAML format", + "summary": "OpenAPI Spec YAML", + "responses": {} + } + }, "/products": { "get": { "description": "List of supported products", @@ -58,7 +79,7 @@ const docTemplate = `{ "/sources": { "get": { "description": "List of supported data sources", - "summary": "List of supported dats sources", + "summary": "List of supported data sources", "responses": { "200": { "description": "Array of dats sources supported by the SP", @@ -77,8 +98,13 @@ const docTemplate = `{ }, "/status/{id}": { "get": { - "description": "List of supported DDO contracts", - "summary": "List of supported DDO contracts", + "security": [ + { + "CurioAuth": [] + } + ], + "description": "Current status of MK20 deal per product", + "summary": "Status of the MK20 deal", "parameters": [ { "type": "string", @@ -112,6 +138,11 @@ const docTemplate = `{ }, "/store": { "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Make a mk20 deal", "consumes": [ "application/json" @@ -218,6 +249,11 @@ const docTemplate = `{ }, "/update/{id}": { "get": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Useful for adding adding additional products and updating PoRep duration", "consumes": [ "application/json" @@ -331,6 +367,11 @@ const docTemplate = `{ }, "/upload/{id}": { "put": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Allows uploading data for deals in a single stream. Suitable for small deals.", "summary": "Upload the deal data", "parameters": [ @@ -382,6 +423,11 @@ const docTemplate = `{ } }, "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Finalizes the serial upload process once data has been uploaded", "consumes": [ "application/json" @@ -494,6 +540,11 @@ const docTemplate = `{ }, "/uploads/finalize/{id}": { "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Finalizes the upload process once all the chunks are uploaded.", "consumes": [ "application/json" @@ -606,6 +657,11 @@ const docTemplate = `{ }, "/uploads/{id}": { "get": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Return a json struct detailing the current status of a deal upload.", "summary": "Status of deal upload", "parameters": [ @@ -651,6 +707,11 @@ const docTemplate = `{ } }, "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", "consumes": [ "application/json" @@ -710,6 +771,11 @@ const docTemplate = `{ }, "/uploads/{id}/{chunkNum}": { "put": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", "summary": "Upload a file chunk", "parameters": [ @@ -779,18 +845,6 @@ const docTemplate = `{ "address.Address": { "type": "object" }, - "cid.Cid": { - "type": "object" - }, - "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { - "type": "integer", - "enum": [ - 0 - ], - "x-enum-varnames": [ - "NoAllocationID" - ] - }, "http.Header": { "type": "object", "additionalProperties": { @@ -815,12 +869,10 @@ const docTemplate = `{ "type": "object", "properties": { "allocation_id": { - "description": "AllocationId represents an aggregated allocation identifier for the deal.", - "allOf": [ - { - "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" - } - ] + "description": "AllocationId represents an allocation identifier for the deal.", + "type": "integer", + "format": "uint64", + "example": 1 }, "contract_address": { "description": "ContractAddress specifies the address of the contract governing the deal", @@ -832,10 +884,8 @@ const docTemplate = `{ }, "contract_verify_method_params": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "duration": { "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", @@ -847,10 +897,8 @@ const docTemplate = `{ }, "notification_payload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "piece_manager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", @@ -883,11 +931,9 @@ const docTemplate = `{ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "allOf": [ - { - "$ref": "#/definitions/cid.Cid" - } - ] + "type": "string", + "format": "cid", + "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -968,11 +1014,10 @@ const docTemplate = `{ ] }, "identifier": { - "description": "Identifier represents a unique identifier for the deal in UUID format.", - "type": "array", - "items": { - "type": "integer" - } + "description": "Identifier represents a unique identifier for the deal in ULID format.", + "type": "string", + "format": "ulid", + "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", @@ -1127,7 +1172,9 @@ const docTemplate = `{ }, "url": { "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", - "type": "string" + "type": "string", + "format": "url", + "example": "http://127.0.0.1:8080/piece/xyz" } } }, @@ -1144,7 +1191,9 @@ const docTemplate = `{ }, "data_set_id": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", - "type": "integer" + "type": "integer", + "format": "uint64", + "example": 0 }, "delete_data_set": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", @@ -1156,17 +1205,21 @@ const docTemplate = `{ }, "extra_data": { "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "piece_ids": { "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { - "type": "integer" - } + "type": "integer", + "format": "uint64" + }, + "example": [ + 0, + 1, + 2 + ] }, "record_keeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", @@ -1186,7 +1239,7 @@ const docTemplate = `{ ] }, "car": { - "description": "Car represents the optional CAR file format, including its metadata and versioning details.", + "description": "Car represents the optional CAR file format.", "allOf": [ { "$ref": "#/definitions/mk20.FormatCar" diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 16d456b55..871eb39a2 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -106,9 +106,13 @@ func AuthMiddleware(db *harmonydb.DB, cfg *config.CurioConfig) func(http.Handler // @title Curio Market 2.0 API // @description Curio market APIs func Router(mdh *MK20DealHandler, domainName string) http.Handler { + SwaggerInfo.BasePath = "/market/mk20" + SwaggerInfo.Host = domainName + SwaggerInfo.Version = version + SwaggerInfo.Schemes = []string{"https"} mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) - mux.Mount("/", APIRouter(mdh, domainName)) + mux.Mount("/", APIRouter(mdh)) mux.Mount("/info", InfoRouter()) return mux } @@ -125,56 +129,67 @@ func Router(mdh *MK20DealHandler, domainName string) http.Handler { // @description - The raw public key bytes (not a human-readable address) // @description - The timestamp, truncated to the nearest hour, formatted in RFC3339 (e.g., 2025-07-15T17:00:00Z) // @description - These two byte slices are joined without any delimiter between them, and the resulting byte array is then hashed using SHA-256. The signature is performed on that hash. -// @security CurioAuth -func APIRouter(mdh *MK20DealHandler, domainName string) http.Handler { - SwaggerInfo.BasePath = "/market/mk20" - SwaggerInfo.Host = fmt.Sprintf("https://%s", domainName) - SwaggerInfo.Version = version +func APIRouter(mdh *MK20DealHandler) http.Handler { mux := chi.NewRouter() mux.Use(dealRateLimitMiddleware()) mux.Use(AuthMiddleware(mdh.db, mdh.cfg)) mux.Method("POST", "/store", http.TimeoutHandler(http.HandlerFunc(mdh.mk20deal), requestTimeout, "request timeout")) mux.Method("GET", "/status/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20status), requestTimeout, "request timeout")) - mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) mux.Method("POST", "/uploads/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStart), requestTimeout, "request timeout")) mux.Method("GET", "/uploads/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UploadStatus), requestTimeout, "request timeout")) mux.Put("/uploads/{id}/{chunkNum}", mdh.mk20UploadDealChunks) mux.Method("POST", "/uploads/finalize/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20FinalizeUpload), requestTimeout, "request timeout")) - mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) - mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) mux.Method("POST", "/update/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20UpdateDeal), requestTimeout, "request timeout")) mux.Method("POST", "/upload/{id}", http.TimeoutHandler(http.HandlerFunc(mdh.mk20SerialUploadFinalize), requestTimeout, "request timeout")) + mux.Method("GET", "/products", http.TimeoutHandler(http.HandlerFunc(mdh.supportedProducts), requestTimeout, "request timeout")) + mux.Method("GET", "/sources", http.TimeoutHandler(http.HandlerFunc(mdh.supportedDataSources), requestTimeout, "request timeout")) + mux.Method("GET", "/contracts", http.TimeoutHandler(http.HandlerFunc(mdh.mk20supportedContracts), requestTimeout, "request timeout")) mux.Put("/upload/{id}", mdh.mk20SerialUpload) return mux } -// InfoRouter serves OpenAPI specs and OpenAPI info +// InfoRouter serves OpenAPI specs UI +// @name info +// @Summary OpenAPI Spec UI +// @description - OpenAPI spec UI for the Market 2.0 APIs +// @Router /info/ [get] +// @BasePath /market/mk20 func InfoRouter() http.Handler { mux := chi.NewRouter() mux.Get("/*", httpSwagger.Handler()) + mux.Get("/swagger.yaml", swaggerYaml) + mux.Get("/swagger.json", swaggerJson) + return mux +} - mux.Get("/swagger.yaml", func(w http.ResponseWriter, r *http.Request) { - swaggerYAML, err := swaggerAssets.ReadFile("swagger.yaml") - if err != nil { - log.Errorw("failed to read swagger.yaml", "err", err) - http.Error(w, "failed to read swagger.yaml", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/x-yaml") - _, _ = w.Write(swaggerYAML) - }) +// @name OpenAPI Spec +// @Summary OpenAPI Spec YAML +// @description - OpenAPI spec for the Market 2.0 APIs in YAML format +// @Router /info/swagger.yaml [get] +func swaggerYaml(w http.ResponseWriter, r *http.Request) { + swaggerYAML, err := swaggerAssets.ReadFile("swagger.yaml") + if err != nil { + log.Errorw("failed to read swagger.yaml", "err", err) + http.Error(w, "failed to read swagger.yaml", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/x-yaml") + _, _ = w.Write(swaggerYAML) +} - mux.Get("/swagger.json", func(w http.ResponseWriter, r *http.Request) { - swaggerJSON, err := swaggerAssets.ReadFile("swagger.json") - if err != nil { - log.Errorw("failed to read swagger.json", "err", err) - http.Error(w, "", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(swaggerJSON) - }) - return mux +// @name OpenAPI Spec +// @Summary OpenAPI Spec JSON +// @description - OpenAPI spec for the Market 2.0 APIs in JSON format +// @Router /info/swagger.json [get] +func swaggerJson(w http.ResponseWriter, r *http.Request) { + swaggerJSON, err := swaggerAssets.ReadFile("swagger.json") + if err != nil { + log.Errorw("failed to read swagger.json", "err", err) + http.Error(w, "", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(swaggerJSON) } // mk20deal handles HTTP requests to process MK20 deals, parses the request body, validates it, and executes the deal logic. @@ -199,6 +214,7 @@ func InfoRouter() http.Handler { // @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" // @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { defer func() { if r := recover(); r != nil { @@ -260,13 +276,14 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { // mk20status handles HTTP requests to fetch the status of a deal by its ID and responding with JSON-encoded results. // @Router /status/{id} [get] -// @Summary List of supported DDO contracts -// @Description List of supported DDO contracts +// @Summary Status of the MK20 deal +// @Description Current status of MK20 deal per product // @BasePath /market/mk20 // @Param id path string true "id" // @Failure 200 {object} mk20.DealProductStatusResponse "the status response for deal products with their respective deal statuses" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" // @Failure 500 {string} string "Internal Server Error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20status(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -373,7 +390,7 @@ func (mdh *MK20DealHandler) supportedProducts(w http.ResponseWriter, r *http.Req // supportedDataSources handles HTTP requests to retrieve the supported data sources in JSON format. // @Router /sources [get] -// @Summary List of supported dats sources +// @Summary List of supported data sources // @Description List of supported data sources // @BasePath /market/mk20 // @Failure 500 {string} string "Internal Server Error" @@ -416,6 +433,7 @@ func (mdh *MK20DealHandler) supportedDataSources(w http.ResponseWriter, r *http. // @Failure 425 {object} mk20.UploadStatusCode "UploadStatusCodeUploadNotStarted indicates that the upload process has not started yet" // @Failure 500 {object} mk20.UploadStatusCode "UploadStatusCodeServerError indicates an internal server error occurred during the upload process, corresponding to status code 500" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -447,6 +465,7 @@ func (mdh *MK20DealHandler) mk20UploadStatus(w http.ResponseWriter, r *http.Requ // @Failure 409 {object} mk20.UploadCode "UploadChunkAlreadyUploaded indicates that the chunk has already been uploaded and cannot be re-uploaded" // @Failure 500 {object} mk20.UploadCode "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") if ct != "application/octet-stream" { @@ -499,6 +518,7 @@ func (mdh *MK20DealHandler) mk20UploadDealChunks(w http.ResponseWriter, r *http. // @Failure 409 {object} mk20.UploadStartCode "UploadStartCodeAlreadyStarted indicates that the upload process has already been initiated and cannot be started again" // @Failure 500 {object} mk20.UploadStartCode "UploadStartCodeServerError indicates an error occurred on the server while processing an upload start request" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Request) { ct := r.Header.Get("Content-Type") if ct != "application/json" { @@ -565,6 +585,7 @@ func (mdh *MK20DealHandler) mk20UploadStart(w http.ResponseWriter, r *http.Reque // @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" // @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -649,6 +670,7 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re // @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" // @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -721,6 +743,7 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques // @Failure 500 {object} mk20.UploadCode "UploadServerError indicates a server-side error occurred during the upload process, represented by the HTTP status code 500" // @Failure 404 {object} mk20.UploadStartCode "UploadStartCodeDealNotFound represents a 404 status indicating the deal was not found during the upload start process" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20SerialUpload(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { @@ -763,6 +786,7 @@ func (mdh *MK20DealHandler) mk20SerialUpload(w http.ResponseWriter, r *http.Requ // @Failure 440 {object} mk20.DealCode "ErrMarketNotEnabled indicates that the market is not enabled for the requested operation" // @Failure 441 {object} mk20.DealCode "ErrDurationTooShort indicates that the provided duration value does not meet the minimum required threshold" // @Failure 400 {string} string "Bad Request - Invalid input or validation error" +// @security CurioAuth func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *http.Request) { idStr := chi.URLParam(r, "id") if idStr == "" { diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 3eac8e605..80ed8b02d 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -26,6 +26,27 @@ } } }, + "/info/": { + "get": { + "description": "- OpenAPI spec UI for the Market 2.0 APIs", + "summary": "OpenAPI Spec UI", + "responses": {} + } + }, + "/info/swagger.json": { + "get": { + "description": "- OpenAPI spec for the Market 2.0 APIs in JSON format", + "summary": "OpenAPI Spec JSON", + "responses": {} + } + }, + "/info/swagger.yaml": { + "get": { + "description": "- OpenAPI spec for the Market 2.0 APIs in YAML format", + "summary": "OpenAPI Spec YAML", + "responses": {} + } + }, "/products": { "get": { "description": "List of supported products", @@ -49,7 +70,7 @@ "/sources": { "get": { "description": "List of supported data sources", - "summary": "List of supported dats sources", + "summary": "List of supported data sources", "responses": { "200": { "description": "Array of dats sources supported by the SP", @@ -68,8 +89,13 @@ }, "/status/{id}": { "get": { - "description": "List of supported DDO contracts", - "summary": "List of supported DDO contracts", + "security": [ + { + "CurioAuth": [] + } + ], + "description": "Current status of MK20 deal per product", + "summary": "Status of the MK20 deal", "parameters": [ { "type": "string", @@ -103,6 +129,11 @@ }, "/store": { "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Make a mk20 deal", "consumes": [ "application/json" @@ -209,6 +240,11 @@ }, "/update/{id}": { "get": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Useful for adding adding additional products and updating PoRep duration", "consumes": [ "application/json" @@ -322,6 +358,11 @@ }, "/upload/{id}": { "put": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Allows uploading data for deals in a single stream. Suitable for small deals.", "summary": "Upload the deal data", "parameters": [ @@ -373,6 +414,11 @@ } }, "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Finalizes the serial upload process once data has been uploaded", "consumes": [ "application/json" @@ -485,6 +531,11 @@ }, "/uploads/finalize/{id}": { "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Finalizes the upload process once all the chunks are uploaded.", "consumes": [ "application/json" @@ -597,6 +648,11 @@ }, "/uploads/{id}": { "get": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Return a json struct detailing the current status of a deal upload.", "summary": "Status of deal upload", "parameters": [ @@ -642,6 +698,11 @@ } }, "post": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Initializes the upload for a deal. Each upload must be initialized before chunks can be uploaded for a deal.", "consumes": [ "application/json" @@ -701,6 +762,11 @@ }, "/uploads/{id}/{chunkNum}": { "put": { + "security": [ + { + "CurioAuth": [] + } + ], "description": "Allows uploading chunks for a deal file. Method can be called in parallel to speed up uploads.", "summary": "Upload a file chunk", "parameters": [ @@ -770,18 +836,6 @@ "address.Address": { "type": "object" }, - "cid.Cid": { - "type": "object" - }, - "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { - "type": "integer", - "enum": [ - 0 - ], - "x-enum-varnames": [ - "NoAllocationID" - ] - }, "http.Header": { "type": "object", "additionalProperties": { @@ -806,12 +860,10 @@ "type": "object", "properties": { "allocation_id": { - "description": "AllocationId represents an aggregated allocation identifier for the deal.", - "allOf": [ - { - "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" - } - ] + "description": "AllocationId represents an allocation identifier for the deal.", + "type": "integer", + "format": "uint64", + "example": 1 }, "contract_address": { "description": "ContractAddress specifies the address of the contract governing the deal", @@ -823,10 +875,8 @@ }, "contract_verify_method_params": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "duration": { "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", @@ -838,10 +888,8 @@ }, "notification_payload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "piece_manager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", @@ -874,11 +922,9 @@ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "allOf": [ - { - "$ref": "#/definitions/cid.Cid" - } - ] + "type": "string", + "format": "cid", + "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -959,11 +1005,10 @@ ] }, "identifier": { - "description": "Identifier represents a unique identifier for the deal in UUID format.", - "type": "array", - "items": { - "type": "integer" - } + "description": "Identifier represents a unique identifier for the deal in ULID format.", + "type": "string", + "format": "ulid", + "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", @@ -1118,7 +1163,9 @@ }, "url": { "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", - "type": "string" + "type": "string", + "format": "url", + "example": "http://127.0.0.1:8080/piece/xyz" } } }, @@ -1135,7 +1182,9 @@ }, "data_set_id": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", - "type": "integer" + "type": "integer", + "format": "uint64", + "example": 0 }, "delete_data_set": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", @@ -1147,17 +1196,21 @@ }, "extra_data": { "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "byte" }, "piece_ids": { "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { - "type": "integer" - } + "type": "integer", + "format": "uint64" + }, + "example": [ + 0, + 1, + 2 + ] }, "record_keeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", @@ -1177,7 +1230,7 @@ ] }, "car": { - "description": "Car represents the optional CAR file format, including its metadata and versioning details.", + "description": "Car represents the optional CAR file format.", "allOf": [ { "$ref": "#/definitions/mk20.FormatCar" diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index fca9d421b..e8a9d2bbb 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -1,14 +1,6 @@ definitions: address.Address: type: object - cid.Cid: - type: object - github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId: - enum: - - 0 - type: integer - x-enum-varnames: - - NoAllocationID http.Header: additionalProperties: items: @@ -26,10 +18,10 @@ definitions: mk20.DDOV1: properties: allocation_id: - allOf: - - $ref: '#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId' - description: AllocationId represents an aggregated allocation identifier for - the deal. + description: AllocationId represents an allocation identifier for the deal. + example: 1 + format: uint64 + type: integer contract_address: description: ContractAddress specifies the address of the contract governing the deal @@ -41,9 +33,8 @@ definitions: contract_verify_method_params: description: ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - items: - type: integer - type: array + format: byte + type: string duration: description: |- Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. @@ -56,9 +47,8 @@ definitions: notification_payload: description: NotificationPayload holds the notification data typically in a serialized byte array format. - items: - type: integer - type: array + format: byte + type: string piece_manager: allOf: - $ref: '#/definitions/address.Address' @@ -77,10 +67,11 @@ definitions: description: Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. piece_cid: - allOf: - - $ref: '#/definitions/cid.Cid' description: PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. + example: bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq + format: cid + type: string source_aggregate: allOf: - $ref: '#/definitions/mk20.DataSourceAggregate' @@ -131,11 +122,11 @@ definitions: - $ref: '#/definitions/mk20.DataSource' description: Data represents the source of piece data and associated metadata. identifier: - description: Identifier represents a unique identifier for the deal in UUID + description: Identifier represents a unique identifier for the deal in ULID format. - items: - type: integer - type: array + example: 01ARZ3NDEKTSV4RRFFQ69G5FAV + format: ulid + type: string products: allOf: - $ref: '#/definitions/mk20.Products' @@ -252,6 +243,8 @@ definitions: type: integer url: description: URL specifies the HTTP endpoint where the piece data can be fetched. + example: http://127.0.0.1:8080/piece/xyz + format: url type: string type: object mk20.PDPV1: @@ -267,6 +260,8 @@ definitions: data_set_id: description: DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. + example: 0 + format: uint64 type: integer delete_data_set: description: |- @@ -280,12 +275,16 @@ definitions: extra_data: description: ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. - items: - type: integer - type: array + format: byte + type: string piece_ids: description: PieceIDs is a list of Piece ids in a proof set. + example: + - 0 + - 1 + - 2 items: + format: uint64 type: integer type: array record_keeper: @@ -303,8 +302,7 @@ definitions: car: allOf: - $ref: '#/definitions/mk20.FormatCar' - description: Car represents the optional CAR file format, including its metadata - and versioning details. + description: Car represents the optional CAR file format. raw: allOf: - $ref: '#/definitions/mk20.FormatBytes' @@ -466,6 +464,21 @@ paths: schema: type: string summary: List of supported DDO contracts + /info/: + get: + description: '- OpenAPI spec UI for the Market 2.0 APIs' + responses: {} + summary: OpenAPI Spec UI + /info/swagger.json: + get: + description: '- OpenAPI spec for the Market 2.0 APIs in JSON format' + responses: {} + summary: OpenAPI Spec JSON + /info/swagger.yaml: + get: + description: '- OpenAPI spec for the Market 2.0 APIs in YAML format' + responses: {} + summary: OpenAPI Spec YAML /products: get: description: List of supported products @@ -491,10 +504,10 @@ paths: description: Internal Server Error schema: type: string - summary: List of supported dats sources + summary: List of supported data sources /status/{id}: get: - description: List of supported DDO contracts + description: Current status of MK20 deal per product parameters: - description: id in: path @@ -515,7 +528,9 @@ paths: description: Internal Server Error schema: type: string - summary: List of supported DDO contracts + security: + - CurioAuth: [] + summary: Status of the MK20 deal /store: post: consumes: @@ -598,6 +613,8 @@ paths: unavailable due to maintenance, corresponding to HTTP status code 503 schema: $ref: '#/definitions/mk20.DealCode' + security: + - CurioAuth: [] summary: Make a mk20 deal /update/{id}: get: @@ -687,6 +704,8 @@ paths: unavailable due to maintenance, corresponding to HTTP status code 503 schema: $ref: '#/definitions/mk20.DealCode' + security: + - CurioAuth: [] summary: Update the deal details of existing deals. /upload/{id}: post: @@ -774,6 +793,8 @@ paths: unavailable due to maintenance, corresponding to HTTP status code 503 schema: $ref: '#/definitions/mk20.DealCode' + security: + - CurioAuth: [] summary: Finalizes the serial upload process put: description: Allows uploading data for deals in a single stream. Suitable for @@ -812,6 +833,8 @@ paths: the upload process, represented by the HTTP status code 500 schema: $ref: '#/definitions/mk20.UploadCode' + security: + - CurioAuth: [] summary: Upload the deal data /uploads/{id}: get: @@ -847,6 +870,8 @@ paths: occurred during the upload process, corresponding to status code 500 schema: $ref: '#/definitions/mk20.UploadStatusCode' + security: + - CurioAuth: [] summary: Status of deal upload post: consumes: @@ -890,6 +915,8 @@ paths: server while processing an upload start request schema: $ref: '#/definitions/mk20.UploadStartCode' + security: + - CurioAuth: [] summary: Starts the upload process /uploads/{id}/{chunkNum}: put: @@ -939,6 +966,8 @@ paths: the upload process, represented by the HTTP status code 500 schema: $ref: '#/definitions/mk20.UploadCode' + security: + - CurioAuth: [] summary: Upload a file chunk /uploads/finalize/{id}: post: @@ -1026,6 +1055,8 @@ paths: unavailable due to maintenance, corresponding to HTTP status code 503 schema: $ref: '#/definitions/mk20.DealCode' + security: + - CurioAuth: [] summary: Finalizes the upload process securityDefinitions: CurioAuth: diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index f4a788d70..425b2ba1b 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -55,15 +55,23 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { } if pdp_complete.Valid { - if pdp_complete.Bool { - ret.Response.DDOV1.State = DealStateComplete + if pdp_complete.Bool && !pdp_error.Valid { + ret.Response.PDPV1.State = DealStateComplete + } + if pdp_complete.Bool && pdp_error.Valid { + ret.Response.PDPV1.State = DealStateFailed + ret.Response.PDPV1.ErrorMsg = pdp_error.String } } if ddo_complete.Valid { - if ddo_complete.Bool { + if ddo_complete.Bool && !ddo_error.Valid { ret.Response.DDOV1.State = DealStateComplete } + if ddo_complete.Bool && ddo_error.Valid { + ret.Response.DDOV1.State = DealStateFailed + ret.Response.DDOV1.ErrorMsg = ddo_error.String + } } if ret.Response.DDOV1.State == DealStateComplete && ret.Response.PDPV1.State == DealStateComplete { diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 242035e23..2abb99de6 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -26,16 +26,16 @@ type PDPV1 struct { DeletePiece bool `json:"delete_piece"` // DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. - DataSetID *uint64 `json:"data_set_id,omitempty"` + DataSetID *uint64 `json:"data_set_id,omitempty" swaggertype:"integer" format:"uint64" example:"0"` // RecordKeeper specifies the record keeper contract address for the new PDP dataset. RecordKeeper string `json:"record_keeper"` // PieceIDs is a list of Piece ids in a proof set. - PieceIDs []uint64 `json:"piece_ids,omitempty"` + PieceIDs []uint64 `json:"piece_ids,omitempty" swaggertype:"array,integer" format:"uint64" example:"0,1,2"` // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. - ExtraData []byte `json:"extra_data,omitempty"` + ExtraData []byte `json:"extra_data,omitempty" swaggertype:"string" format:"byte"` } func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { diff --git a/market/mk20/types.go b/market/mk20/types.go index 92c5c7d19..e6aef6bb7 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -13,8 +13,8 @@ import ( // Deal represents a structure defining the details and components of a specific deal in the system. type Deal struct { - // Identifier represents a unique identifier for the deal in UUID format. - Identifier ulid.ULID `json:"identifier"` + // Identifier represents a unique identifier for the deal in ULID format. + Identifier ulid.ULID `json:"identifier" swaggertype:"string" format:"ulid" example:"01ARZ3NDEKTSV4RRFFQ69G5FAV"` // Client wallet string for the deal Client string `json:"client"` @@ -41,7 +41,7 @@ type Products struct { type DataSource struct { // PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. - PieceCID cid.Cid `json:"piece_cid"` + PieceCID cid.Cid `json:"piece_cid" swaggertype:"string" format:"cid" example:"bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"` // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` @@ -62,7 +62,7 @@ type DataSource struct { // PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. type PieceDataFormat struct { - // Car represents the optional CAR file format, including its metadata and versioning details. + // Car represents the optional CAR file format. Car *FormatCar `json:"car,omitempty"` // Aggregate holds a reference to the aggregated format of piece data. @@ -107,7 +107,7 @@ type DataSourceHTTP struct { type HttpUrl struct { // URL specifies the HTTP endpoint where the piece data can be fetched. - URL string `json:"url"` + URL string `json:"url" swaggertype:"string" format:"url" example:"http://127.0.0.1:8080/piece/xyz"` // HTTPHeaders represents the HTTP headers associated with the URL. Headers http.Header `json:"headers"` From aafe7a77c0ef03b59dd1bb4cd5cecea12a7c2333 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 4 Sep 2025 14:23:33 -0500 Subject: [PATCH 32/55] niceties --- lib/commcidv2/commcidv2.go | 41 ++- market/mk20/tsclient/README.md | 156 ++++++++- market/mk20/tsclient/examples/basic-usage.ts | 54 ++- .../mk20/tsclient/examples/pdpv1-workflow.ts | 3 +- .../mk20/tsclient/examples/product-types.ts | 195 +++++++++++ .../mk20/tsclient/examples/streaming-pdp.ts | 36 ++ market/mk20/tsclient/package-lock.json | 31 +- market/mk20/tsclient/package.json | 3 +- market/mk20/tsclient/src/client.ts | 319 +++++++++++++++++- market/mk20/tsclient/src/index.ts | 1 + market/mk20/tsclient/src/streaming.ts | 276 +++++++++++++++ market/mk20/tsclient/tests/client.test.ts | 2 +- .../tests/examples.basic-usage.test.ts | 45 +++ .../tests/examples.product-types.test.ts | 46 +++ .../tests/examples.streaming-pdp.test.ts | 43 +++ 15 files changed, 1178 insertions(+), 73 deletions(-) create mode 100644 market/mk20/tsclient/examples/product-types.ts create mode 100644 market/mk20/tsclient/examples/streaming-pdp.ts create mode 100644 market/mk20/tsclient/src/streaming.ts create mode 100644 market/mk20/tsclient/tests/examples.basic-usage.test.ts create mode 100644 market/mk20/tsclient/tests/examples.product-types.test.ts create mode 100644 market/mk20/tsclient/tests/examples.streaming-pdp.test.ts diff --git a/lib/commcidv2/commcidv2.go b/lib/commcidv2/commcidv2.go index aa804fa7e..5000dbf9f 100644 --- a/lib/commcidv2/commcidv2.go +++ b/lib/commcidv2/commcidv2.go @@ -176,6 +176,26 @@ func (cp *CommP) PCidV2() cid.Cid { func (cp *CommP) Digest() []byte { return cp.digest } +func IsPieceCidV2(c cid.Cid) bool { + if c.Type() != uint64(multicodec.Raw) { + return false + } + + decoded, err := multihash.Decode(c.Hash()) + if err != nil { + return false + } + + if decoded.Code != uint64(multicodec.Fr32Sha256Trunc254Padbintree) { + return false + } + + if len(decoded.Digest) < 34 { + return false + } + + return true +} func PieceCidV2FromV1(v1PieceCid cid.Cid, payloadsize uint64) (cid.Cid, error) { decoded, err := multihash.Decode(v1PieceCid.Hash()) if err != nil { @@ -212,24 +232,3 @@ func PieceCidV2FromV1(v1PieceCid cid.Cid, payloadsize uint64) (cid.Cid, error) { return c.PCidV2(), nil } - -func IsPieceCidV2(c cid.Cid) bool { - if c.Type() != uint64(multicodec.Raw) { - return false - } - - decoded, err := multihash.Decode(c.Hash()) - if err != nil { - return false - } - - if decoded.Code != uint64(multicodec.Fr32Sha256Trunc254Padbintree) { - return false - } - - if len(decoded.Digest) < 34 { - return false - } - - return true -} diff --git a/market/mk20/tsclient/README.md b/market/mk20/tsclient/README.md index 0f87175a2..76d2b2b3e 100644 --- a/market/mk20/tsclient/README.md +++ b/market/mk20/tsclient/README.md @@ -35,9 +35,7 @@ npm run build ```typescript import { MarketClient, PieceCidUtils } from '@curio/market-client'; -const client = new MarketClient({ - basePath: 'http://localhost:8080/market/mk20' -}); +const client = new MarketClient({ serverUrl: 'http://localhost:8080' }); // Get supported contracts const contracts = await client.getContracts(); @@ -73,9 +71,107 @@ const uploadStatus = await client.getUploadStatus('deal-id'); const blobs = [new Blob(['file content'])]; const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); -// Convert CID v1 to piece CID v2 -const cidV1 = CID.create(1, 0x55, hash); -const pieceCidV2 = await PieceCidUtils.pieceCidV2FromV1(cidV1, dataSize); +// Convenience wrappers for common workflows (includes automatic chunked upload) +const result = await client.submitPDPv1DealWithUpload({ + blobs: [new Blob(['file content'])], + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...' +}); + +// DDO deals with custom duration (includes automatic chunked upload) +const ddoResult = await client.submitDDOV1DealWithUpload({ + blobs: [new Blob(['file content'])], + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...', + // Optional lifespan (epochs); defaults to 518400 if omitted + lifespan: 600000 +}); + +// Results include upload statistics +console.log('Uploaded chunks:', result.uploadedChunks); +console.log('Uploaded bytes:', result.uploadedBytes); +``` + +## Streaming PDP (no upfront data section) + +Create a deal without a `data` section, stream data using `uploadChunk`, compute the piece CID while streaming, then finalize with the computed `data`: + +```typescript +import { Client, MarketClientConfig } from '@curio/market-client'; + +const config: MarketClientConfig = { serverUrl: 'http://localhost:8080' }; +const client = new Client(config); + +// Create the streaming helper (defaults to 1MB chunks) +const spdp = client.streamingPDP({ + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...', + // chunkSize: 2 * 1024 * 1024, // optional +}); + +// Begin: submits deal without data and initializes chunked upload +await spdp.begin(); + +// Stream bytes (these are uploaded as chunks and hashed for CID) +spdp.write(new TextEncoder().encode('hello ')); +spdp.write(new TextEncoder().encode('world')); + +// Commit: flushes remaining chunk, computes piece CID, and finalizes with data +const { id, pieceCid, totalSize } = await spdp.commit(); +console.log({ id, pieceCid, totalSize }); +``` + +## Product Types + +The client supports three main product types for different use cases: + +### PDPv1 (Proof of Data Possession) +Used for creating datasets and proving data possession: +```typescript +products: { + pdpV1: { + createDataSet: true, // Create new dataset + addPiece: true, // Add piece to dataset + recordKeeper: 'provider-address', + pieceIds: [123, 456, 789] // Piece IDs for each individual blob + }, + retrievalV1: { + announcePayload: true, // Announce to IPNI + announcePiece: true, // Announce piece info + indexing: true // Enable retrieval + } +} +``` + +### DDOv1 (Direct Data Onboarding) +Used for direct data onboarding with contract verification: +```typescript +products: { + ddoV1: { + duration: 518400, // Typically chosen per-deal (lifespan) + provider: { address: 'provider-address' }, + contractAddress: '0x...', + contractVerifyMethod: 'verifyDeal' + }, + retrievalV1: { + announcePayload: true, + announcePiece: true, + indexing: true + } +} +``` + +### RetrievalV1 +Configures retrieval behavior and indexing: +```typescript +retrievalV1: { + announcePayload: true, // Announce payload to IPNI + announcePiece: true, // Announce piece information + indexing: true // Index for CID-based retrieval +} ``` ## API Endpoints @@ -91,6 +187,54 @@ const pieceCidV2 = await PieceCidUtils.pieceCidV2FromV1(cidV1, dataSize); - `POST /uploads/finalize/{id}` - Finalize chunked upload - `GET /uploads/{id}` - Get upload status +## Automatic Chunked Upload + +The convenience wrappers automatically handle chunked uploads after deal submission: + +- **Automatic Processing**: After submitting a deal, all blobs are automatically uploaded in chunks +- **Configurable Chunk Size**: Uses 1MB chunks by default for optimal performance +- **Progress Tracking**: Provides detailed logging of upload progress +- **Complete Workflow**: Handles initialization, chunking, upload, and finalization +- **Upload Statistics**: Returns total chunks and bytes uploaded +- **Simple & Reliable**: Sequential uploads ensure data integrity and predictable behavior + +```typescript +const result = await client.submitPDPv1DealWithUpload({ + blobs: [blob1, blob2, blob3], + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...' +}); + +// The result includes upload statistics +console.log('Uploaded chunks:', result.uploadedChunks); // Total number of chunks +console.log('Uploaded bytes:', result.uploadedBytes); // Total bytes uploaded +``` + +## Piece ID Calculation + +The client automatically calculates unique piece IDs for each blob in a deal: + +- **Individual Blob Piece IDs**: Each blob gets a unique piece ID based on its content hash and size +- **Deterministic**: The same blob content will always generate the same piece ID +- **Consistent**: Both PDPv1 and DDOv1 deals use the same piece ID calculation method +- **Returned**: Piece IDs are included in the deal creation and returned by convenience wrappers + +```typescript +// Each blob gets its own piece ID +const result = await client.submitPDPv1DealWithUpload({ + blobs: [blob1, blob2, blob3], + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...' +}); + +console.log('Piece IDs:', result.pieceIds); // [123, 456, 789] +console.log('Blob 1 ā†’ Piece ID:', result.pieceIds[0]); // 123 +console.log('Blob 2 ā†’ Piece ID:', result.pieceIds[1]); // 456 +console.log('Blob 3 ā†’ Piece ID:', result.pieceIds[2]); // 789 +``` + ## Piece CID Computation The client includes utilities for computing Filecoin piece CIDs using the [js-multiformats library](https://github.com/multiformats/js-multiformats): diff --git a/market/mk20/tsclient/examples/basic-usage.ts b/market/mk20/tsclient/examples/basic-usage.ts index 84141bb1e..51afdf336 100644 --- a/market/mk20/tsclient/examples/basic-usage.ts +++ b/market/mk20/tsclient/examples/basic-usage.ts @@ -1,8 +1,8 @@ -import { Client, MarketClientConfig, Deal, DataSource, Products, DDOV1 } from '../src'; +import { Client, MarketClientConfig, Deal, DataSource, Products, DDOV1, RetrievalV1 } from '../src'; // Example configuration const config: MarketClientConfig = { - basePath: 'http://localhost:8080/market/mk20', + serverUrl: 'http://localhost:8080', // Optional: Add custom headers headers: { 'Authorization': 'Bearer your-token-here' @@ -44,8 +44,8 @@ async function exampleUsage() { } } as DataSource, products: { - ddo_v1: { - duration: 518400, // Minimum duration in epochs + ddoV1: { + duration: 518400, // Typical lifespan value (epochs) provider: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, contractAddress: '0x1234567890123456789012345678901234567890', contractVerifyMethod: 'verifyDeal', @@ -53,7 +53,12 @@ async function exampleUsage() { pieceManager: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, notificationAddress: 'f1abcdefghijklmnopqrstuvwxyz123456789', notificationPayload: [] - } as DDOV1 + } as DDOV1, + retrievalV1: { + announcePayload: true, // Announce payload to IPNI + announcePiece: true, // Announce piece information to IPNI + indexing: true // Index for CID-based retrieval + } as RetrievalV1 } as Products }; @@ -83,4 +88,41 @@ async function uploadDataExample(dealId: string, data: number[]) { } } -export { exampleUsage, uploadDataExample }; +// Example: Demonstrate piece ID calculation for individual blobs +async function pieceIdCalculationExample() { + try { + console.log('šŸ” Piece ID Calculation Example'); + console.log('Calculating piece IDs for individual blobs...\n'); + + // Create mock blobs with different content + const mockBlobs = [ + new Blob(['file1 content'], { type: 'text/plain' }), + new Blob(['file2 content'], { type: 'text/plain' }), + new Blob(['file3 content'], { type: 'text/plain' }) + ]; + + // Use the convenience wrapper to see piece IDs + const result = await client.submitPDPv1DealWithUpload({ + blobs: mockBlobs, + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + provider: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', + contractAddress: '0x1234567890123456789012345678901234567890' + }); + + console.log('šŸ“‹ Deal and Upload Results:'); + console.log('UUID:', result.uuid); + console.log('Total Size:', result.totalSize, 'bytes'); + console.log('Deal ID:', result.dealId); + console.log('Piece CID:', result.pieceCid); + console.log('Uploaded Chunks:', result.uploadedChunks); + console.log('Uploaded Bytes:', result.uploadedBytes); + + return result; + + } catch (error) { + console.error('āŒ Piece ID calculation example failed:', error); + throw error; + } +} + +export { exampleUsage, uploadDataExample, pieceIdCalculationExample }; diff --git a/market/mk20/tsclient/examples/pdpv1-workflow.ts b/market/mk20/tsclient/examples/pdpv1-workflow.ts index 028327369..9d1bd19d4 100644 --- a/market/mk20/tsclient/examples/pdpv1-workflow.ts +++ b/market/mk20/tsclient/examples/pdpv1-workflow.ts @@ -32,7 +32,8 @@ async function pdpv1CompleteWorkflowExample() { uuid: result.uuid, totalSize: result.totalSize, dealId: result.dealId, - pieceCid: result.pieceCid + pieceCid: result.pieceCid, + pieceIds: result.pieceIds }); // Upload data in chunks using the actual blobs diff --git a/market/mk20/tsclient/examples/product-types.ts b/market/mk20/tsclient/examples/product-types.ts new file mode 100644 index 000000000..33d69dcd1 --- /dev/null +++ b/market/mk20/tsclient/examples/product-types.ts @@ -0,0 +1,195 @@ +import { Client, MarketClientConfig, Deal, Products, PDPV1, DDOV1, RetrievalV1 } from '../src'; + +const config: MarketClientConfig = { + serverUrl: 'http://localhost:8080', + headers: { 'Authorization': 'Bearer your-token-here' } +}; + +const client = new Client(config); + +// Example 1: PDPv1 Product (Proof of Data Possession) +async function pdpv1ProductExample() { + console.log('šŸ” PDPv1 Product Example'); + console.log('Used for: Creating datasets, adding pieces, and proving data possession\n'); + + const pdpv1Deal: Deal = { + identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + data: { + piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + format: { raw: {} }, + source_httpput: { raw_size: 1024 * 1024 } + } as any, + products: { + pdpV1: { + createDataSet: true, // Create a new dataset + addPiece: true, // Add piece to the dataset + dataSetId: undefined, // Not needed when creating dataset + recordKeeper: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', + extraData: [], // Additional data for verification + pieceIds: [0], // Initial piece ID + deleteDataSet: false, // Don't delete dataset + deletePiece: false // Don't delete piece + } as PDPV1, + retrievalV1: { + announcePayload: true, // Announce to IPNI + announcePiece: true, // Announce piece info + indexing: true // Enable CID-based retrieval + } as RetrievalV1 + } as Products + }; + + console.log('PDPv1 Deal Structure:', JSON.stringify(pdpv1Deal.products, null, 2)); + return pdpv1Deal; +} + +// Example 2: DDOv1 Product (Direct Data Onboarding) +async function ddov1ProductExample() { + console.log('šŸ“„ DDOv1 Product Example'); + console.log('Used for: Direct data onboarding with contract verification\n'); + + const ddov1Deal: Deal = { + identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + data: { + piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + format: { raw: {} }, + source_httpput: { raw_size: 1024 * 1024 } + } as any, + products: { + ddoV1: { + duration: 518400, // Typical lifespan value (epochs) + provider: { address: 'f1provider123456789abcdefghijklmnopqrstuvwxyz' }, + contractAddress: '0x1234567890123456789012345678901234567890', + contractVerifyMethod: 'verifyDeal', + contractVerifyMethodParams: [], + pieceManager: { address: 'f1provider123456789abcdefghijklmnopqrstuvwxyz' }, + notificationAddress: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + notificationPayload: [] + } as DDOV1, + retrievalV1: { + announcePayload: true, // Announce to IPNI + announcePiece: true, // Announce piece info + indexing: true // Enable CID-based retrieval + } as RetrievalV1 + } as Products + }; + + console.log('DDOv1 Deal Structure:', JSON.stringify(ddov1Deal.products, null, 2)); + return ddov1Deal; +} + +// Example 3: RetrievalV1 Product (Retrieval Configuration) +async function retrievalV1ProductExample() { + console.log('šŸ” RetrievalV1 Product Example'); + console.log('Used for: Configuring retrieval behavior and indexing\n'); + + const retrievalConfig: RetrievalV1 = { + announcePayload: true, // Announce payload to IPNI + announcePiece: true, // Announce piece information to IPNI + indexing: true // Index for CID-based retrieval + }; + + console.log('RetrievalV1 Configuration:', JSON.stringify(retrievalConfig, null, 2)); + return retrievalConfig; +} + +// Example 4: Using the convenience wrappers +async function convenienceWrapperExample() { + console.log('šŸš€ Convenience Wrapper Examples'); + console.log('Using the simplified methods for common workflows\n'); + + // Create mock blobs + const mockBlobs = [ + new Blob(['file1 content'], { type: 'text/plain' }), + new Blob(['file2 content'], { type: 'text/plain' }) + ]; + + try { + // PDPv1 workflow + console.log('\nšŸ“‹ PDPv1 Workflow:'); + const pdpResult = await client.submitPDPv1DealWithUpload({ + blobs: mockBlobs, + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + provider: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', + contractAddress: '0x1234567890123456789012345678901234567890' + }); + + console.log('PDPv1 Result:', { + uuid: pdpResult.uuid, + totalSize: pdpResult.totalSize, + dealId: pdpResult.dealId, + pieceCid: pdpResult.pieceCid, + uploadedChunks: pdpResult.uploadedChunks, + uploadedBytes: pdpResult.uploadedBytes + }); + + // Show blob to piece ID mapping + console.log('šŸ“ Blob to Piece ID Mapping:'); + mockBlobs.forEach((blob, index) => { + console.log(` Blob ${index + 1} (${blob.size} bytes)`); + }); + + // DDOv1 workflow + console.log('\nšŸ“‹ DDOv1 Workflow:'); + const ddoResult = await client.submitDDOV1DealWithUpload({ + blobs: mockBlobs, + client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', + provider: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', + contractAddress: '0x1234567890123456789012345678901234567890', + lifespan: 518400 + }); + + console.log('DDOv1 Result:', { + uuid: ddoResult.uuid, + totalSize: ddoResult.totalSize, + dealId: ddoResult.dealId, + pieceCid: ddoResult.pieceCid, + uploadedChunks: ddoResult.uploadedChunks, + uploadedBytes: ddoResult.uploadedBytes + }); + + // Show blob to piece ID mapping + console.log('šŸ“ Blob to Piece ID Mapping:'); + mockBlobs.forEach((blob, index) => { + console.log(` Blob ${index + 1} (${blob.size} bytes)`); + }); + + } catch (error) { + console.error('āŒ Error in convenience wrapper example:', error); + } +} + +// Main function to run all examples +async function runAllProductExamples() { + console.log('šŸŽÆ Market Client Product Types Examples\n'); + console.log('=====================================\n'); + + try { + // Show product structures + await pdpv1ProductExample(); + console.log('\n' + '='.repeat(50) + '\n'); + + await ddov1ProductExample(); + console.log('\n' + '='.repeat(50) + '\n'); + + await retrievalV1ProductExample(); + console.log('\n' + '='.repeat(50) + '\n'); + + // Show convenience wrappers + await convenienceWrapperExample(); + + console.log('\nāœ… All product examples completed successfully!'); + + } catch (error) { + console.error('āŒ Error running product examples:', error); + } +} + +export { + pdpv1ProductExample, + ddov1ProductExample, + retrievalV1ProductExample, + convenienceWrapperExample, + runAllProductExamples +}; diff --git a/market/mk20/tsclient/examples/streaming-pdp.ts b/market/mk20/tsclient/examples/streaming-pdp.ts new file mode 100644 index 000000000..5a1a841c1 --- /dev/null +++ b/market/mk20/tsclient/examples/streaming-pdp.ts @@ -0,0 +1,36 @@ +import { MarketClientConfig } from '../src'; +import { StreamingPDP } from '../src/streaming'; + +// Example usage (now using the strongly-typed StreamingPDP from src) +async function example() { + const config: MarketClientConfig = { + serverUrl: 'http://localhost:8080', + } as MarketClientConfig; + const client = new (require('../src').Client)(config); + const spdp = new StreamingPDP(client, { + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...', + }); + + await spdp.begin(); + + // Simulate streaming writes + spdp.write(new TextEncoder().encode('hello ')); + spdp.write(new TextEncoder().encode('world')); + + const res = await spdp.commit(); + console.log('Streaming PDP completed:', res); +} + +// Only run wenshen executed directly (ts-node/node), not when imported +if (require.main === module) { + example().catch(err => { + console.error('Streaming PDP example failed:', err); + process.exit(1); + }); +} + +export { StreamingPDP }; + + diff --git a/market/mk20/tsclient/package-lock.json b/market/mk20/tsclient/package-lock.json index 0f0a9be21..0a9d1573a 100644 --- a/market/mk20/tsclient/package-lock.json +++ b/market/mk20/tsclient/package-lock.json @@ -9,10 +9,9 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "@types/uuid": "^10.0.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", - "uuid": "^11.1.0" + "ulid": "^2.3.0" }, "devDependencies": { "@openapitools/openapi-generator-cli": "^2.7.0", @@ -1434,12 +1433,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", - "license": "MIT" - }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -5592,6 +5585,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ulid": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.4.0.tgz", + "integrity": "sha512-fIRiVTJNcSRmXKPZtGzFQv9WRrZ3M9eoptl/teFJvjOzmpU+/K/JH6HZ8deBfb5vMEpicJcLn7JmvdknlMq7Zg==", + "license": "MIT", + "bin": { + "ulid": "bin/cli.js" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -5647,19 +5649,6 @@ "dev": true, "license": "MIT" }, - "node_modules/uuid": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", - "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/esm/bin/uuid" - } - }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index c27b9f2ca..9d2ab1211 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -33,10 +33,9 @@ "typescript": "^5.0.0" }, "dependencies": { - "@types/uuid": "^10.0.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", - "uuid": "^11.1.0" + "ulid": "^2.3.0" }, "engines": { "node": ">=18.0.0" diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index f22b1afc1..cb0d865e4 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -1,10 +1,11 @@ -import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources } from '../generated'; -import { v4 as uuidv4 } from 'uuid'; +import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources, Mk20Products, Mk20PDPV1, Mk20RetrievalV1, Mk20DDOV1, Mk20DataSource } from '../generated'; +import { ulid } from 'ulid'; import { Configuration } from '../generated/runtime'; import { Mk20StartUpload } from '../generated/models/Mk20StartUpload'; +import { StreamingPDP } from './streaming'; -export interface MarketClientConfig extends ConfigurationParameters { - basePath: string; +export interface MarketClientConfig extends Omit { + serverUrl: string; // e.g. http://localhost:8080 } /** @@ -260,8 +261,43 @@ export class PieceCidUtils { export class MarketClient { private api: DefaultApi; + /** + * Create a MarketClient instance. + * @param config - Configuration object + * @param config.serverUrl - Base server URL, e.g. http://localhost:8080 + * @param config.headers - Optional default headers to send with every request + * @param config.fetchApi - Optional custom fetch implementation + */ constructor(config: MarketClientConfig) { - this.api = new DefaultApi(new Configuration(config)); + const basePath = `${config.serverUrl.replace(/\/$/, '')}/market/mk20`; + const runtimeConfig = { ...config, basePath } as ConfigurationParameters; + this.api = new DefaultApi(new Configuration(runtimeConfig)); + } + + /** + * Factory: create a StreamingPDP helper bound to this client instance + */ + /** + * Create a StreamingPDP helper bound to this client instance. + * @param params - Streaming parameters + * @param params.client - Client wallet address + * @param params.provider - Provider wallet address + * @param params.contractAddress - Verification contract address + * @param params.chunkSize - Optional chunk size in bytes (default 1MB) + */ + streamingPDP(params: { client: string; provider: string; contractAddress: string; chunkSize?: number }): StreamingPDP { + return new StreamingPDP(this, params); + } + + /** + * Convert a ULID string (26-char Crockford base32) into a 16-byte array + */ + private ulidToBytes(ulidString: string): number[] { + var bytes: number[] = []; + for (let i = 0; i < ulidString.length; i++) { + bytes.push(ulidString.charCodeAt(i)); + } + return bytes; } /** @@ -303,6 +339,10 @@ export class MarketClient { /** * Get deal status by ID */ + /** + * Get deal status by ID. + * @param id - Deal identifier (string ULID returned from submit wrappers) + */ async getStatus(id: string): Promise { try { const response = await this.api.statusIdGet({ id }); @@ -315,6 +355,10 @@ export class MarketClient { /** * Submit a new deal */ + /** + * Submit a new deal. + * @param deal - Deal payload matching Mk20Deal schema + */ async submitDeal(deal: Mk20Deal): Promise { try { const response = await this.api.storePost({ body: deal }); @@ -324,10 +368,45 @@ export class MarketClient { } } + + + /** + * Calculate piece ID for an individual blob based on its content + * @param blob - The blob to calculate piece ID for + * @returns Promise - A unique piece ID for this blob + */ + private async calculateBlobPieceId(blob: Blob): Promise { + // Create a hash from the blob's content to generate a unique piece ID + const arrayBuffer = await blob.arrayBuffer(); + const uint8Array = new Uint8Array(arrayBuffer); + + let hash = 0; + for (let i = 0; i < uint8Array.length; i++) { + hash = ((hash << 5) - hash) + uint8Array[i]; + hash = hash & hash; // Convert to 32-bit integer + } + + // Add size to the hash to make it more unique + hash = ((hash << 5) - hash) + blob.size; + hash = hash & hash; + + // Ensure positive and within reasonable bounds + return Math.abs(hash) % 1000000; // Keep within 6 digits + } + /** * Simple convenience wrapper for PDPv1 deals with chunked upload * Takes blobs and required addresses, computes piece_cid, and returns a UUID identifier */ + /** + * Convenience wrapper for PDPv1 deals with chunked upload. + * @param params - Input parameters + * @param params.blobs - Data to upload as an array of blobs + * @param params.client - Client wallet address + * @param params.provider - Provider wallet address + * @param params.contractAddress - Verification contract address + * @returns Upload metadata including uuid, pieceCid, and stats + */ async submitPDPv1DealWithUpload(params: { blobs: Blob[]; client: string; @@ -339,23 +418,167 @@ export class MarketClient { dealId: number; uploadId: string; pieceCid: string; + uploadedChunks: number; + uploadedBytes: number; + }> { + try { + const { blobs, client, provider, contractAddress } = params; + + // Calculate total size from blobs + const totalSize = blobs.reduce((sum, blob) => sum + blob.size, 0); + + // Generate a ULID for the deal identifier returned to the caller + const uuid = ulid(); + // TODO make a streaming example with no data block until finalize, use uploadSerial + + // Compute piece_cid from blobs using our utility (uses WebCrypto in browser, Node crypto fallback) + const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + + + // Create deal with required addresses + var deal: Mk20Deal = { + // Use the generated UUID as the deal identifier + identifier: this.ulidToBytes(uuid), + client, + data: { + piece_cid: pieceCid, + format: { raw: {} }, + source_httpput: { + raw_size: totalSize + } + } as Mk20DataSource, + products: { + pdpV1: { + createDataSet: true, // Create a new dataset for this deal + addPiece: true, // Add the piece to the dataset + dataSetId: undefined, // Not needed when creating dataset + recordKeeper: provider, // Use provider as record keeper + extraData: [], // No extra data needed + pieceIds: undefined, // Piece IDs (on chain) not available for new content. + deleteDataSet: false, + deletePiece: false + } as Mk20PDPV1, + retrievalV1: { + announcePayload: true, // Announce payload to IPNI + announcePiece: true, // Announce piece information to IPNI + indexing: true // Index for CID-based retrieval + } as Mk20RetrievalV1 + } as Mk20Products + }; + + // Submit the deal + const dealId = await this.submitDeal(deal); + + // Initialize chunked upload + const startUpload: Mk20StartUpload = { + rawSize: totalSize, + chunkSize: 1024 * 1024 // 1MB chunks + }; + + const uploadInitResult = await this.initializeChunkedUpload(uuid, startUpload); + + // Automatically upload all blobs in chunks + console.log(`šŸ“¤ Starting automatic chunked upload of ${blobs.length} blobs...`); + const chunkSize = 1024 * 1024; // 1MB chunks + let totalChunks = 0; + let uploadedBytes = 0; + + for (const [blobIndex, blob] of blobs.entries()) { + const blobSize = blob.size; + const blobChunks = Math.ceil(blobSize / chunkSize); + + console.log(` Uploading blob ${blobIndex + 1}/${blobs.length} (${blobSize} bytes, ${blobChunks} chunks)...`); + + for (let i = 0; i < blobSize; i += chunkSize) { + const chunk = blob.slice(i, i + chunkSize); + const chunkNum = totalChunks.toString(); + + // Convert blob chunk to array of numbers for upload + const chunkArray = new Uint8Array(await chunk.arrayBuffer()); + const chunkNumbers = Array.from(chunkArray); + + console.log(` Uploading chunk ${chunkNum + 1} (${chunkNumbers.length} bytes)...`); + await this.uploadChunk(uuid, chunkNum, chunkNumbers); + + totalChunks++; + uploadedBytes += chunkNumbers.length; + } + } + + // Finalize the upload + console.log('šŸ”’ Finalizing chunked upload...'); + const finalizeResult = await this.finalizeChunkedUpload(uuid, deal); + console.log(`āœ… Upload finalized: ${finalizeResult}`); + + return { + uuid, + totalSize, + dealId, + uploadId: uuid, + pieceCid, + uploadedChunks: totalChunks, + uploadedBytes + }; + + } catch (error) { + throw new Error(`Failed to submit PDPv1 deal with upload: ${error}`); + } + } + + /** + * Simple convenience wrapper for DDO deals with chunked upload + * Takes blobs and required addresses, computes piece_cid, and returns a UUID identifier + */ + /** + * Convenience wrapper for DDOv1 deals with chunked upload. + * @param params - Input parameters + * @param params.blobs - Data to upload as an array of blobs + * @param params.client - Client wallet address + * @param params.provider - Provider wallet address + * @param params.contractAddress - Verification contract address + * @param params.lifespan - Optional deal lifespan in epochs (defaults to 518400) + * @returns Upload metadata including uuid, pieceCid, and stats + */ + async submitDDOV1DealWithUpload(params: { + blobs: Blob[]; + client: string; + provider: string; + contractAddress: string; + lifespan?: number; + }): Promise<{ + uuid: string; + totalSize: number; + dealId: number; + uploadId: string; + pieceCid: string; + pieceIds: number[]; + uploadedChunks: number; + uploadedBytes: number; }> { try { const { blobs, client, provider, contractAddress } = params; + const duration = params.lifespan ?? 518400; // Calculate total size from blobs const totalSize = blobs.reduce((sum, blob) => sum + blob.size, 0); - // Generate a proper UUID v4 identifier - const uuid = uuidv4(); + // Generate a ULID for the deal identifier returned to the caller + const uuid = ulid(); - // Compute piece_cid from blobs using our utility + // Compute piece_cid from blobs using our utility (uses WebCrypto in browser, Node crypto fallback) const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + // Calculate piece IDs for each individual blob + const pieceIds: number[] = []; + for (const blob of blobs) { + const pieceId = await this.calculateBlobPieceId(blob); + pieceIds.push(pieceId); + } + // Create deal with required addresses const deal: Mk20Deal = { // Use the generated UUID as the deal identifier - identifier: Array.from(uuid.replace(/-/g, '').match(/.{1,2}/g)!.map(hex => parseInt(hex, 16))), + identifier: this.ulidToBytes(uuid), client, data: { piece_cid: pieceCid, @@ -365,8 +588,8 @@ export class MarketClient { } } as any, products: { - pdp_v1: { - duration: 518400, // Minimum duration + ddoV1: { + duration, // Deal duration in epochs provider: { address: provider }, contractAddress, contractVerifyMethod: 'verifyDeal', @@ -374,8 +597,13 @@ export class MarketClient { pieceManager: { address: provider }, notificationAddress: client, notificationPayload: [] - } as any - } as any + } as Mk20DDOV1, + retrievalV1: { + announcePayload: true, // Announce payload to IPNI + announcePiece: true, // Announce piece information to IPNI + indexing: true // Index for CID-based retrieval + } as Mk20RetrievalV1 + } as Mk20Products }; // Submit the deal @@ -389,22 +617,63 @@ export class MarketClient { const uploadInitResult = await this.initializeChunkedUpload(uuid, startUpload); + // Automatically upload all blobs in chunks + console.log(`šŸ“¤ Starting automatic chunked upload of ${blobs.length} blobs...`); + const chunkSize = 1024 * 1024; // 1MB chunks + let totalChunks = 0; + let uploadedBytes = 0; + + for (const [blobIndex, blob] of blobs.entries()) { + const blobSize = blob.size; + const blobChunks = Math.ceil(blobSize / chunkSize); + + console.log(` Uploading blob ${blobIndex + 1}/${blobs.length} (${blobSize} bytes, ${blobChunks} chunks)...`); + + for (let i = 0; i < blobSize; i += chunkSize) { + const chunk = blob.slice(i, i + chunkSize); + const chunkNum = totalChunks.toString(); + + // Convert blob chunk to array of numbers for upload + const chunkArray = new Uint8Array(await chunk.arrayBuffer()); + const chunkNumbers = Array.from(chunkArray); + + console.log(` Uploading chunk ${chunkNum + 1} (${chunkNumbers.length} bytes)...`); + await this.uploadChunk(uuid, chunkNum, chunkNumbers); + + totalChunks++; + uploadedBytes += chunkNumbers.length; + } + } + + // Finalize the upload + console.log('šŸ”’ Finalizing chunked upload...'); + const finalizeResult = await this.finalizeChunkedUpload(uuid); + console.log(`āœ… Upload finalized: ${finalizeResult}`); + return { uuid, totalSize, dealId, uploadId: uuid, - pieceCid + pieceCid, + pieceIds, + uploadedChunks: totalChunks, + uploadedBytes }; } catch (error) { - throw new Error(`Failed to submit PDPv1 deal with upload: ${error}`); + throw new Error(`Failed to submit DDOv1 deal with upload: ${error}`); } } /** * Upload deal data */ + /** + * Upload all data in a single request (for small deals). + * @param id - Deal identifier + * @param data - Entire data payload as an array of bytes + */ async uploadData(id: string, data: Array): Promise { try { await this.api.uploadIdPut({ id, body: data }); @@ -418,6 +687,11 @@ export class MarketClient { * @param id - Deal identifier * @param startUpload - Upload initialization data */ + /** + * Initialize chunked upload for a deal. + * @param id - Deal identifier + * @param startUpload - Upload init payload (chunkSize, rawSize) + */ async initializeChunkedUpload(id: string, startUpload: Mk20StartUpload): Promise { try { const result = await this.api.uploadsIdPost({ id, data: startUpload }); @@ -433,6 +707,12 @@ export class MarketClient { * @param chunkNum - Chunk number * @param data - Chunk data */ + /** + * Upload one chunk for a deal. + * @param id - Deal identifier + * @param chunkNum - Chunk index as string (0-based) + * @param data - Chunk data bytes + */ async uploadChunk(id: string, chunkNum: string, data: Array): Promise { try { const result = await this.api.uploadsIdChunkNumPut({ id, chunkNum, data }); @@ -447,6 +727,11 @@ export class MarketClient { * @param id - Deal identifier * @param deal - Optional deal data for finalization */ + /** + * Finalize a chunked upload. + * @param id - Deal identifier + * @param deal - Optional deal payload to finalize with + */ async finalizeChunkedUpload(id: string, deal?: any): Promise { try { const result = await this.api.uploadsFinalizeIdPost({ id, body: deal }); @@ -460,6 +745,10 @@ export class MarketClient { * Get upload status for a deal * @param id - Deal identifier */ + /** + * Get upload status for a deal. + * @param id - Deal identifier + */ async getUploadStatus(id: string): Promise { try { return await this.api.uploadsIdGet({ id }); diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts index a1abfa0d6..37af1acf9 100644 --- a/market/mk20/tsclient/src/index.ts +++ b/market/mk20/tsclient/src/index.ts @@ -25,6 +25,7 @@ export type { MarketClientConfig } from './client'; // Export piece CID utilities export { PieceCidUtils } from './client'; +export { StreamingPDP } from './streaming'; // Re-export configuration types export type { Configuration } from '../generated'; diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts new file mode 100644 index 000000000..4180b4e45 --- /dev/null +++ b/market/mk20/tsclient/src/streaming.ts @@ -0,0 +1,276 @@ +import { MarketClient as Client } from './client'; +import type { Mk20Deal as Deal, Mk20Products as Products, Mk20PDPV1 as PDPV1, Mk20RetrievalV1 as RetrievalV1, Mk20DataSource, Mk20PieceDataFormat } from '../generated'; +import { ulid } from 'ulid'; + +namespace StreamingCommP { + const NODE_SIZE = 32; + const NODE_LOG2_SIZE = 5; + + function calculateTreeHeight(boxSize: number): number { + let leadingZeros = 0; + let temp = boxSize; + while (temp > 0) { + temp = temp >>> 1; + leadingZeros++; + } + leadingZeros = 64 - leadingZeros; + let treeHeight = 63 - leadingZeros - NODE_LOG2_SIZE; + if (countOnes(boxSize) !== 1) treeHeight++; + return treeHeight; + } + + function countOnes(n: number): number { + let count = 0; + while (n > 0) { + count += n & 1; + n = n >>> 1; + } + return count; + } + + function varintSize(value: number): number { + if (value < 0x80) return 1; + if (value < 0x4000) return 2; + if (value < 0x200000) return 3; + if (value < 0x10000000) return 4; + if (value < 0x800000000) return 5; + if (value < 0x40000000000) return 6; + if (value < 0x2000000000000) return 7; + if (value < 0x100000000000000) return 8; + return 9; + } + + function putVarint(buf: Uint8Array, offset: number, value: number): number { + let n = 0; + while (value >= 0x80) { + buf[offset + n] = (value & 0x7f) | 0x80; + value = value >>> 7; + n++; + } + buf[offset + n] = value & 0x7f; + return n + 1; + } + + function bytesToCidString(bytes: Uint8Array): string { + const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; + let result = ''; + let value = 0; + let bits = 0; + for (let i = 0; i < bytes.length; i++) { + value = (value << 8) | bytes[i]; + bits += 8; + while (bits >= 5) { + result += base32Chars[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) { + result += base32Chars[(value << (5 - bits)) & 31]; + } + return `b${result}`; + } + + export function pieceCidV2FromDigest(payloadSize: number, digest: Uint8Array): string { + let psz = payloadSize; + if (psz < 127) psz = 127; + const boxSize = Math.ceil((psz + 126) / 127) * 128; + const treeHeight = calculateTreeHeight(boxSize); + const payloadPadding = ((1 << (treeHeight - 2)) * 127) - payloadSize; + + const prefix = new Uint8Array([0x01, 0x55, 0x91, 0x20]); + const ps = varintSize(payloadPadding); + const bufSize = prefix.length + 1 + ps + 1 + NODE_SIZE; + const buf = new Uint8Array(bufSize); + + let n = 0; + buf.set(prefix, n); n += prefix.length; + buf[n] = ps + 1 + NODE_SIZE; n++; + n += putVarint(buf, n, payloadPadding); + buf[n] = treeHeight; n++; + buf.set(digest, n); + + return bytesToCidString(buf); + } +} + +/** + * StreamingPDP provides a streaming workflow to create a deal without a data section, + * push data via chunked upload, compute the piece CID while streaming, and finalize. + */ +export class StreamingPDP { + private client: Client; + private id: string; + private identifierBytes: number[]; + private totalSize = 0; + private hashBuffers: Uint8Array[] = []; + private deal: Deal | undefined; + private clientAddr: string; + private providerAddr: string; + private contractAddress: string; + private chunkSize: number; + private buffer: number[] = []; + private nextChunkNum = 0; + private uploadedBytes = 0; + private totalChunks = 0; + + /** + * @param client - Market client instance + * @param opts - Streaming options + * @param opts.client - Client wallet address + * @param opts.provider - Provider wallet address + * @param opts.contractAddress - Verification contract address + * @param opts.chunkSize - Optional chunk size in bytes (default 1MB) + */ + constructor(client: Client, opts: { client: string; provider: string; contractAddress: string; chunkSize?: number }) { + this.client = client; + this.clientAddr = opts.client; + this.providerAddr = opts.provider; + this.contractAddress = opts.contractAddress; + this.chunkSize = opts.chunkSize ?? 1024 * 1024; + this.id = ulid(); + this.identifierBytes = Array.from(this.id).map(c => c.charCodeAt(0)).slice(0, 16); + while (this.identifierBytes.length < 16) this.identifierBytes.push(0); + } + + /** + * Begin the streaming deal by submitting a deal without data and initializing upload. + */ + async begin(): Promise { + const products: Products = { + pdpV1: { + createDataSet: true, + addPiece: true, + recordKeeper: this.providerAddr, + extraData: [], + pieceIds: undefined, + deleteDataSet: false, + deletePiece: false, + } as PDPV1, + retrievalV1: { + announcePayload: true, + announcePiece: true, + indexing: true, + } as RetrievalV1, + } as Products; + + const deal: Deal = { + identifier: this.identifierBytes, + client: this.clientAddr, + products, + } as Deal; + + this.deal = deal; + await this.client.submitDeal(deal); + + await this.client.initializeChunkedUpload(this.id, { chunkSize: this.chunkSize }); + } + + /** + * Write a chunk of data into the stream. This will upload full chunks immediately + * and buffer the remainder until the next write or commit. + * @param chunk - Data bytes to write + */ + write(chunk: Uint8Array | Buffer): void { + const u8 = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); + this.totalSize += u8.length; + // Cross-env hashing fallback: store chunks for hashing at commit using WebCrypto or Node crypto + this.hashBuffers.push(u8); + + let idx = 0; + if (this.buffer.length > 0) { + const needed = this.chunkSize - this.buffer.length; + const take = Math.min(needed, u8.length); + for (let i = 0; i < take; i++) this.buffer.push(u8[idx + i]); + idx += take; + if (this.buffer.length === this.chunkSize) { + const toSend = this.buffer.slice(0, this.chunkSize); + void this.uploadChunkNow(toSend); + this.buffer = []; + } + } + + while (u8.length - idx >= this.chunkSize) { + const sub = u8.subarray(idx, idx + this.chunkSize); + const toSend = Array.from(sub); + void this.uploadChunkNow(toSend); + idx += this.chunkSize; + } + + for (let i = idx; i < u8.length; i++) this.buffer.push(u8[i]); + } + + /** + * Finalize the streaming deal: flush remaining data, compute piece CID, and finalize. + * @returns Object containing id (ULID), pieceCid, and totalSize + */ + async commit(): Promise<{ id: string; pieceCid: string; totalSize: number }> { + if (!this.deal) throw new Error('StreamingPDP not started. Call begin() first.'); + + if (this.buffer.length > 0) { + const toSend = this.buffer.slice(); + await this.uploadChunkNow(toSend); + this.buffer = []; + } + + const digest = await this.computeDigest(); + const pieceCid = StreamingCommP.pieceCidV2FromDigest(this.totalSize, digest); + + const dataSource: Mk20DataSource = { + pieceCid: pieceCid as unknown as object, + format: { raw: {} } as Mk20PieceDataFormat, + sourceHttpput: { raw_size: this.totalSize } as unknown as object, + }; + + const finalizedDeal: Deal = { + ...this.deal, + data: dataSource, + } as Deal; + + await this.client.finalizeChunkedUpload(this.id, finalizedDeal); + + return { id: this.id, pieceCid, totalSize: this.totalSize }; + } + + /** + * Upload a single chunk immediately. + * @param data - Chunk bytes + */ + private async uploadChunkNow(data: number[]): Promise { + const chunkNum = String(this.nextChunkNum); + await this.client.uploadChunk(this.id, chunkNum, data); + this.nextChunkNum++; + this.uploadedBytes += data.length; + this.totalChunks++; + } + + /** + * Compute SHA-256 digest of all streamed bytes using WebCrypto in browsers + * and Node crypto as a fallback in Node environments. + */ + private async computeDigest(): Promise { + const total = this.hashBuffers.reduce((n, b) => n + b.length, 0); + const all = new Uint8Array(total); + let offset = 0; + for (const b of this.hashBuffers) { + all.set(b, offset); + offset += b.length; + } + + if (typeof globalThis !== 'undefined' && (globalThis as any).crypto && (globalThis as any).crypto.subtle) { + const h = await (globalThis as any).crypto.subtle.digest('SHA-256', all); + return new Uint8Array(h); + } + + try { + const nodeCrypto = await import('crypto'); + const hasher = nodeCrypto.createHash('sha256'); + hasher.update(Buffer.from(all)); + return new Uint8Array(hasher.digest()); + } catch { + // Last resort: simple JS fallback (not streaming) using built-in SubtleCrypto if available, else throw + throw new Error('No available crypto implementation to compute SHA-256 digest in this environment'); + } + } +} + + diff --git a/market/mk20/tsclient/tests/client.test.ts b/market/mk20/tsclient/tests/client.test.ts index 59a263ac0..1f31bc250 100644 --- a/market/mk20/tsclient/tests/client.test.ts +++ b/market/mk20/tsclient/tests/client.test.ts @@ -19,7 +19,7 @@ describe('MarketClient', () => { beforeEach(() => { const config: MarketClientConfig = { - basePath: 'http://localhost:8080/market/mk20', + serverUrl: 'http://localhost:8080', } as MarketClientConfig; client = new MarketClient(config); diff --git a/market/mk20/tsclient/tests/examples.basic-usage.test.ts b/market/mk20/tsclient/tests/examples.basic-usage.test.ts new file mode 100644 index 000000000..7f0093b1a --- /dev/null +++ b/market/mk20/tsclient/tests/examples.basic-usage.test.ts @@ -0,0 +1,45 @@ +// Tests covering examples/basic-usage.ts + +jest.mock('../generated', () => ({ + DefaultApi: jest.fn().mockImplementation(() => ({ + contractsGet: jest.fn().mockResolvedValue({ contracts: ['0xabc'] }), + productsGet: jest.fn().mockResolvedValue({ ddo_v1: true, pdp_v1: true }), + sourcesGet: jest.fn().mockResolvedValue({ http: true, aggregate: true }), + statusIdGet: jest.fn().mockResolvedValue({ identifier: 'id', status: 'active' }), + storePost: jest.fn().mockResolvedValue(200), + uploadIdPut: jest.fn().mockResolvedValue(undefined), + uploadsIdPost: jest.fn().mockResolvedValue(200), + uploadsIdChunkNumPut: jest.fn().mockResolvedValue(200), + uploadsFinalizeIdPost: jest.fn().mockResolvedValue(200), + })), +})); + +import { exampleUsage, uploadDataExample, pieceIdCalculationExample } from '../examples/basic-usage'; +import { PieceCidUtils } from '../src'; + +describe('examples/basic-usage.ts', () => { + beforeAll(() => { + jest.spyOn(PieceCidUtils, 'computePieceCidV2').mockResolvedValue('btestcid'); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it('runs exampleUsage without errors', async () => { + await expect(exampleUsage()).resolves.not.toThrow(); + }); + + it('runs uploadDataExample without errors', async () => { + await expect(uploadDataExample('deal-id', [1, 2, 3])).resolves.not.toThrow(); + }); + + it('runs pieceIdCalculationExample and returns results', async () => { + const res = await pieceIdCalculationExample(); + expect(res).toBeDefined(); + expect(res.dealId).toBe(200); + expect(res.pieceCid).toBe('btestcid'); + }); +}); + + diff --git a/market/mk20/tsclient/tests/examples.product-types.test.ts b/market/mk20/tsclient/tests/examples.product-types.test.ts new file mode 100644 index 000000000..53277348c --- /dev/null +++ b/market/mk20/tsclient/tests/examples.product-types.test.ts @@ -0,0 +1,46 @@ +// Tests covering examples/product-types.ts + +jest.mock('../generated', () => ({ + DefaultApi: jest.fn().mockImplementation(() => ({ + storePost: jest.fn().mockResolvedValue(200), + contractsGet: jest.fn().mockResolvedValue({ contracts: ['0xabc'] }), + productsGet: jest.fn().mockResolvedValue({ ddo_v1: true, pdp_v1: true }), + sourcesGet: jest.fn().mockResolvedValue({ http: true, aggregate: true }), + })), +})); + +import { pdpv1ProductExample, ddov1ProductExample, retrievalV1ProductExample, convenienceWrapperExample } from '../examples/product-types'; +import { PieceCidUtils } from '../src'; + +describe('examples/product-types.ts', () => { + beforeAll(() => { + jest.spyOn(PieceCidUtils, 'computePieceCidV2').mockResolvedValue('btestcid'); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it('pdpv1ProductExample returns a deal structure', async () => { + const deal = await pdpv1ProductExample(); + expect(deal.products).toBeDefined(); + expect(deal.products?.pdpV1).toBeDefined(); + }); + + it('ddov1ProductExample returns a deal structure', async () => { + const deal = await ddov1ProductExample(); + expect(deal.products).toBeDefined(); + expect(deal.products?.ddoV1).toBeDefined(); + }); + + it('retrievalV1ProductExample returns a config', async () => { + const cfg = await retrievalV1ProductExample(); + expect(cfg.indexing).toBe(true); + }); + + it('convenienceWrapperExample runs without errors', async () => { + await expect(convenienceWrapperExample()).resolves.not.toThrow(); + }); +}); + + diff --git a/market/mk20/tsclient/tests/examples.streaming-pdp.test.ts b/market/mk20/tsclient/tests/examples.streaming-pdp.test.ts new file mode 100644 index 000000000..1b60b3965 --- /dev/null +++ b/market/mk20/tsclient/tests/examples.streaming-pdp.test.ts @@ -0,0 +1,43 @@ +// Tests covering examples/streaming-pdp.ts and StreamingPDP helper + +jest.mock('../generated', () => ({ + DefaultApi: jest.fn().mockImplementation(() => ({ + storePost: jest.fn().mockResolvedValue(200), + uploadsIdPost: jest.fn().mockResolvedValue(200), + uploadsIdChunkNumPut: jest.fn().mockResolvedValue(200), + uploadsFinalizeIdPost: jest.fn().mockResolvedValue(200), + })), +})); + +import { MarketClientConfig, Client, StreamingPDP, PieceCidUtils } from '../src'; + +describe('streaming-pdp example and helper', () => { + beforeAll(() => { + jest.spyOn(PieceCidUtils, 'computePieceCidV2').mockResolvedValue('btestcid'); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it('streams data via StreamingPDP and finalizes', async () => { + const config: MarketClientConfig = { serverUrl: 'http://localhost:8080' } as MarketClientConfig; + const client = new Client(config); + const spdp = client.streamingPDP({ + client: 'f1client...', + provider: 'f1provider...', + contractAddress: '0x...', + }); + + await spdp.begin(); + spdp.write(new TextEncoder().encode('hello ')); + spdp.write(new TextEncoder().encode('world')); + const res = await spdp.commit(); + + expect(res.id).toBeDefined(); + expect(res.pieceCid).toMatch(/^b/); + expect(res.totalSize).toBeGreaterThan(0); + }); +}); + + From bcb975e2de22c17769797de75c321d90a962c093 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 4 Sep 2025 14:51:12 -0500 Subject: [PATCH 33/55] minor fixes --- market/mk20/tsclient/src/client.ts | 29 +++++++++++++++++++++------ market/mk20/tsclient/src/streaming.ts | 4 ++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index cb0d865e4..dba3e9bdb 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -43,9 +43,8 @@ export class PieceCidUtils { offset += uint8Array.length; } - // Compute SHA256 hash - const hash = await crypto.subtle.digest('SHA-256', concatenatedData); - const hashArray = new Uint8Array(hash); + // Compute SHA256 hash (works in browser and Node) + const hashArray = await this.computeSha256(concatenatedData); // Create CommP using the exact Go algorithm const commP = this.newSha2CommP(totalSize, hashArray); @@ -59,6 +58,24 @@ export class PieceCidUtils { } } + /** + * Compute SHA-256 digest cross-environment (browser WebCrypto or Node crypto) + */ + private static async computeSha256(data: Uint8Array): Promise { + if (typeof globalThis !== 'undefined' && (globalThis as any).crypto && (globalThis as any).crypto.subtle) { + const h = await (globalThis as any).crypto.subtle.digest('SHA-256', data); + return new Uint8Array(h); + } + try { + const nodeCrypto = await import('crypto'); + const hasher = nodeCrypto.createHash('sha256'); + hasher.update(Buffer.from(data)); + return new Uint8Array(hasher.digest()); + } catch { + throw new Error('No available crypto implementation to compute SHA-256 digest in this environment'); + } + } + /** * NewSha2CommP - exact port of Go function * @param payloadSize - Size of the payload in bytes @@ -301,7 +318,7 @@ export class MarketClient { } /** - * Get supported DDO contracts + * Get supported contracts */ async getContracts(): Promise { try { @@ -431,7 +448,7 @@ export class MarketClient { const uuid = ulid(); // TODO make a streaming example with no data block until finalize, use uploadSerial - // Compute piece_cid from blobs using our utility (uses WebCrypto in browser, Node crypto fallback) + // Compute piece_cid from blobs using our utility (WebCrypto SubtleCrypto) const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); @@ -565,7 +582,7 @@ export class MarketClient { // Generate a ULID for the deal identifier returned to the caller const uuid = ulid(); - // Compute piece_cid from blobs using our utility (uses WebCrypto in browser, Node crypto fallback) + // Compute piece_cid from blobs using our utility (WebCrypto SubtleCrypto) const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); // Calculate piece IDs for each individual blob diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts index 4180b4e45..2109e7dcd 100644 --- a/market/mk20/tsclient/src/streaming.ts +++ b/market/mk20/tsclient/src/streaming.ts @@ -166,8 +166,8 @@ export class StreamingPDP { } /** - * Write a chunk of data into the stream. This will upload full chunks immediately - * and buffer the remainder until the next write or commit. + * Write a chunk of data into the stream. This uploads full chunks immediately + * and buffers any remainder until the next write or commit. * @param chunk - Data bytes to write */ write(chunk: Uint8Array | Buffer): void { From f01531747b51b7eb6415f5d0a8f87f722c345cf9 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Fri, 5 Sep 2025 18:54:46 -0500 Subject: [PATCH 34/55] picked-up the changes, e2e test --- market/mk20/tsclient/examples/basic-usage.ts | 12 +-- .../mk20/tsclient/examples/pdpv1-workflow.ts | 5 +- .../mk20/tsclient/examples/product-types.ts | 18 ++-- .../tsclient/examples/unpkg-end-to-end.ts | 100 ++++++++++++++++++ .../mk20/tsclient/examples/upload-methods.ts | 78 +------------- market/mk20/tsclient/package.json | 2 +- market/mk20/tsclient/src/client.ts | 50 +++++++-- market/mk20/tsclient/src/streaming.ts | 6 +- market/mk20/tsclient/tests/client.test.ts | 2 +- 9 files changed, 167 insertions(+), 106 deletions(-) create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end.ts diff --git a/market/mk20/tsclient/examples/basic-usage.ts b/market/mk20/tsclient/examples/basic-usage.ts index 51afdf336..989103279 100644 --- a/market/mk20/tsclient/examples/basic-usage.ts +++ b/market/mk20/tsclient/examples/basic-usage.ts @@ -32,16 +32,16 @@ async function exampleUsage() { // Example: Submit a deal console.log('\nSubmitting a deal...'); const deal: Deal = { - identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], // Example identifier + identifier: '01H0EXAMPLEULIDIDENTIFIER00000000', // Example ULID string client: 'f1abcdefghijklmnopqrstuvwxyz123456789', data: { - piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + pieceCid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', format: { raw: {} }, - source_httpput: { + sourceHttpput: { raw_size: 1024 * 1024 // 1MB - } + } as unknown as object } as DataSource, products: { ddoV1: { @@ -49,10 +49,10 @@ async function exampleUsage() { provider: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, contractAddress: '0x1234567890123456789012345678901234567890', contractVerifyMethod: 'verifyDeal', - contractVerifyMethodParams: [], + contractVerifyMethodParams: '', pieceManager: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, notificationAddress: 'f1abcdefghijklmnopqrstuvwxyz123456789', - notificationPayload: [] + notificationPayload: '' } as DDOV1, retrievalV1: { announcePayload: true, // Announce payload to IPNI diff --git a/market/mk20/tsclient/examples/pdpv1-workflow.ts b/market/mk20/tsclient/examples/pdpv1-workflow.ts index 9d1bd19d4..fd7495a83 100644 --- a/market/mk20/tsclient/examples/pdpv1-workflow.ts +++ b/market/mk20/tsclient/examples/pdpv1-workflow.ts @@ -1,7 +1,7 @@ import { Client, MarketClientConfig } from '../src'; const config: MarketClientConfig = { - basePath: 'http://localhost:8080/market/mk20', + serverUrl: 'http://localhost:8080', headers: { 'Authorization': 'Bearer your-token-here' } }; @@ -33,7 +33,8 @@ async function pdpv1CompleteWorkflowExample() { totalSize: result.totalSize, dealId: result.dealId, pieceCid: result.pieceCid, - pieceIds: result.pieceIds + uploadedChunks: result.uploadedChunks, + uploadedBytes: result.uploadedBytes }); // Upload data in chunks using the actual blobs diff --git a/market/mk20/tsclient/examples/product-types.ts b/market/mk20/tsclient/examples/product-types.ts index 33d69dcd1..c315f6650 100644 --- a/market/mk20/tsclient/examples/product-types.ts +++ b/market/mk20/tsclient/examples/product-types.ts @@ -13,12 +13,12 @@ async function pdpv1ProductExample() { console.log('Used for: Creating datasets, adding pieces, and proving data possession\n'); const pdpv1Deal: Deal = { - identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + identifier: '01H0EXAMPLEULIDIDENTIFIER00000000', client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', data: { - piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + pieceCid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', format: { raw: {} }, - source_httpput: { raw_size: 1024 * 1024 } + sourceHttpput: { raw_size: 1024 * 1024 } as unknown as object } as any, products: { pdpV1: { @@ -26,7 +26,7 @@ async function pdpv1ProductExample() { addPiece: true, // Add piece to the dataset dataSetId: undefined, // Not needed when creating dataset recordKeeper: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', - extraData: [], // Additional data for verification + extraData: '', // Additional data for verification pieceIds: [0], // Initial piece ID deleteDataSet: false, // Don't delete dataset deletePiece: false // Don't delete piece @@ -49,12 +49,12 @@ async function ddov1ProductExample() { console.log('Used for: Direct data onboarding with contract verification\n'); const ddov1Deal: Deal = { - identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + identifier: '01H0EXAMPLEULIDIDENTIFIER00000000', client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', data: { - piece_cid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', + pieceCid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', format: { raw: {} }, - source_httpput: { raw_size: 1024 * 1024 } + sourceHttpput: { raw_size: 1024 * 1024 } as unknown as object } as any, products: { ddoV1: { @@ -62,10 +62,10 @@ async function ddov1ProductExample() { provider: { address: 'f1provider123456789abcdefghijklmnopqrstuvwxyz' }, contractAddress: '0x1234567890123456789012345678901234567890', contractVerifyMethod: 'verifyDeal', - contractVerifyMethodParams: [], + contractVerifyMethodParams: '', pieceManager: { address: 'f1provider123456789abcdefghijklmnopqrstuvwxyz' }, notificationAddress: 'f1client123456789abcdefghijklmnopqrstuvwxyz', - notificationPayload: [] + notificationPayload: '' } as DDOV1, retrievalV1: { announcePayload: true, // Announce to IPNI diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end.ts b/market/mk20/tsclient/examples/unpkg-end-to-end.ts new file mode 100644 index 000000000..3ec6aa97e --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end.ts @@ -0,0 +1,100 @@ +// Set before running: +// PDP_URL=https://andyserver.thepianoexpress.com +// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet +// PDP_PROVIDER=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // provider wallet +// PDP_TOKEN=your-token-here +// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 +import { Client, MarketClientConfig, PieceCidUtils } from '../src'; + +async function downloadFromUnpkg(url: string): Promise { + const res = await fetch(url); + if (!res.ok) throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); + const buf = await res.arrayBuffer(); + return new Uint8Array(buf); +} + +async function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function run() { + const config: MarketClientConfig = { + serverUrl: process.env.PDP_URL || 'http://localhost:8080', + headers: process.env.PDP_TOKEN ? { Authorization: `Bearer ${process.env.PDP_TOKEN}` } : undefined, + } as MarketClientConfig; + const client = new (require('../src').Client)(config); + + const clientAddr = process.env.PDP_CLIENT || 'f1client...'; + const providerAddr = process.env.PDP_PROVIDER || 'f1provider...'; + const contractAddress = process.env.PDP_CONTRACT || '0x0000000000000000000000000000000000000000'; + + const targetUrl = 'https://unpkg.com/react@18/umd/react.production.min.js'; + console.log(`ā¬‡ļø Downloading: ${targetUrl}`); + const bytes = await downloadFromUnpkg(targetUrl); + console.log(` Downloaded ${bytes.length} bytes`); + + // Compute piece CID locally for retrieval + const blob = new Blob([bytes], { type: 'application/octet-stream' }); + const pieceCid = await PieceCidUtils.computePieceCidV2([blob]); + console.log(`šŸ”— Computed piece CID: ${pieceCid}`); + + console.log('šŸ“Ø Submitting PDPv1 deal and uploading via helper'); + const res = await client.submitPDPv1DealWithUpload({ + blobs: [blob], + client: clientAddr, + provider: providerAddr, + contractAddress, + }); + const uploadId = res.uploadId; + + console.log('ā³ Polling deal status until complete/failed'); + for (let i = 0; i < 120; i++) { // up to ~10 minutes with 5s interval + const status = await client.getStatus(uploadId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') break; + await sleep(5000); + } + + console.log('šŸ“¦ Retrieving piece via market server'); + try { + const base = config.serverUrl.replace(/\/$/, ''); + const url = `${base}/piece/${pieceCid}`; + const r = await fetch(url); + console.log(` retrieval HTTP ${r.status}`); + if (r.ok) { + const retrieved = new Uint8Array(await r.arrayBuffer()); + console.log(` retrieved ${retrieved.length} bytes`); + } + } catch (e) { + console.warn(' Retrieval attempt failed:', (e as Error).message); + } + + console.log('šŸ—‘ļø Requesting deletion (set delete flags via update)'); + await client.updateDeal(uploadId, { + client: clientAddr, + products: { pdpV1: { deletePiece: true, deleteDataSet: true } }, + } as any); + + console.log('ā³ Polling deal status post-deletion'); + for (let i = 0; i < 24; i++) { // up to ~2 minutes + const status = await client.getStatus(uploadId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') break; + await sleep(5000); + } + + console.log('āœ… Example finished'); +} + +if (require.main === module) { + run().catch(err => { + console.error('Example failed:', err); + process.exit(1); + }); +} + +export {}; diff --git a/market/mk20/tsclient/examples/upload-methods.ts b/market/mk20/tsclient/examples/upload-methods.ts index d83037a71..304260e69 100644 --- a/market/mk20/tsclient/examples/upload-methods.ts +++ b/market/mk20/tsclient/examples/upload-methods.ts @@ -2,27 +2,16 @@ import { Client, MarketClientConfig, StartUpload } from '../src'; // Example configuration const config: MarketClientConfig = { - basePath: 'http://localhost:8080/market/mk20', + serverUrl: 'https://andyserver.thepianoexpress.com', headers: { - 'Authorization': 'Bearer your-token-here' + //'Authorization': 'Bearer your-token-here' } }; // Create client instance const client = new Client(config); -// Example 1: Single upload (suitable for small deals) -async function singleUploadExample(dealId: string, data: number[]) { - try { - console.log(`Uploading ${data.length} bytes for deal ${dealId}...`); - await client.uploadData(dealId, data); - console.log('Single upload completed successfully'); - } catch (error) { - console.error('Single upload failed:', error); - } -} - -// Example 2: Chunked upload (suitable for large deals) +// Example: Chunked upload (suitable for large deals) async function chunkedUploadExample(dealId: string, largeData: number[], chunkSize: number = 1024 * 1024) { try { console.log(`Starting chunked upload for deal ${dealId}...`); @@ -67,62 +56,7 @@ async function chunkedUploadExample(dealId: string, largeData: number[], chunkSi } } -// Example 3: Parallel chunk uploads for better performance -async function parallelChunkUploadExample(dealId: string, largeData: number[], chunkSize: number = 1024 * 1024) { - try { - console.log(`Starting parallel chunked upload for deal ${dealId}...`); - - // Step 1: Initialize the upload - const startUpload: StartUpload = { - rawSize: largeData.length, - chunkSize: chunkSize - }; - - await client.initializeChunkedUpload(dealId, startUpload); - console.log('Upload initialized'); - - // Step 2: Prepare all chunks - const chunks: Array<{ chunkNum: string; data: number[] }> = []; - for (let i = 0; i < largeData.length; i += chunkSize) { - const chunk = largeData.slice(i, i + chunkSize); - const chunkNum = Math.floor(i / chunkSize).toString(); - chunks.push({ chunkNum, data: chunk }); - } - - console.log(`Uploading ${chunks.length} chunks in parallel...`); - - // Step 3: Upload chunks in parallel (with concurrency limit) - const concurrencyLimit = 5; // Limit concurrent requests - const results: Array<{ chunkNum: string; result: number }> = []; - - for (let i = 0; i < chunks.length; i += concurrencyLimit) { - const batch = chunks.slice(i, i + concurrencyLimit); - const batchPromises = batch.map(async ({ chunkNum, data }) => { - const result = await client.uploadChunk(dealId, chunkNum, data); - return { chunkNum, result }; - }); - - const batchResults = await Promise.all(batchPromises); - results.push(...batchResults); - - console.log(`Completed batch ${Math.floor(i / concurrencyLimit) + 1}/${Math.ceil(chunks.length / concurrencyLimit)}`); - } - - console.log(`All ${results.length} chunks uploaded successfully`); - - // Step 4: Finalize the upload - console.log('Finalizing upload...'); - const finalizeResult = await client.finalizeChunkedUpload(dealId); - console.log('Upload finalized with result:', finalizeResult); - - console.log('Parallel chunked upload completed successfully'); - - } catch (error) { - console.error('Parallel chunked upload failed:', error); - } -} - -// Example 4: Monitor upload progress +// Example: Monitor upload progress async function monitoredUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024) { try { console.log(`Starting monitored upload for deal ${dealId}...`); @@ -166,7 +100,7 @@ async function monitoredUploadExample(dealId: string, data: number[], chunkSize: } } -// Example 5: Error handling and retry logic +// Example: Error handling and retry logic async function robustUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024, maxRetries: number = 3) { try { console.log(`Starting robust upload for deal ${dealId}...`); @@ -221,9 +155,7 @@ async function robustUploadExample(dealId: string, data: number[], chunkSize: nu } export { - singleUploadExample, chunkedUploadExample, - parallelChunkUploadExample, monitoredUploadExample, robustUploadExample }; diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 9d2ab1211..7789b639c 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -6,7 +6,7 @@ "types": "dist/index.d.ts", "scripts": { "build": "npm run generate && npm run compile", - "generate": "openapi-generator-cli generate -i ../http/swagger.json -g typescript-fetch -o ./generated --additional-properties=supportsES6=true,typescriptThreePlus=true", + "generate": "openapi-generator-cli generate -i ../http/swagger.json -g typescript-fetch -o ./generated --additional-properties=supportsES6=true,typescriptThreePlus=true --skip-validate-spec", "compile": "tsc", "clean": "rm -rf dist generated", "test": "jest", diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index dba3e9bdb..e4a60c992 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -455,14 +455,14 @@ export class MarketClient { // Create deal with required addresses var deal: Mk20Deal = { // Use the generated UUID as the deal identifier - identifier: this.ulidToBytes(uuid), + identifier: uuid, client, data: { - piece_cid: pieceCid, + pieceCid: pieceCid, format: { raw: {} }, - source_httpput: { + sourceHttpput: { raw_size: totalSize - } + } as unknown as object, } as Mk20DataSource, products: { pdpV1: { @@ -470,7 +470,7 @@ export class MarketClient { addPiece: true, // Add the piece to the dataset dataSetId: undefined, // Not needed when creating dataset recordKeeper: provider, // Use provider as record keeper - extraData: [], // No extra data needed + extraData: '', // No extra data pieceIds: undefined, // Piece IDs (on chain) not available for new content. deleteDataSet: false, deletePiece: false @@ -595,14 +595,14 @@ export class MarketClient { // Create deal with required addresses const deal: Mk20Deal = { // Use the generated UUID as the deal identifier - identifier: this.ulidToBytes(uuid), + identifier: uuid, client, data: { - piece_cid: pieceCid, + pieceCid: pieceCid, format: { raw: {} }, - source_httpput: { + sourceHttpput: { raw_size: totalSize - } + } as unknown as object, } as any, products: { ddoV1: { @@ -610,10 +610,10 @@ export class MarketClient { provider: { address: provider }, contractAddress, contractVerifyMethod: 'verifyDeal', - contractVerifyMethodParams: [], + contractVerifyMethodParams: '', pieceManager: { address: provider }, notificationAddress: client, - notificationPayload: [] + notificationPayload: '' } as Mk20DDOV1, retrievalV1: { announcePayload: true, // Announce payload to IPNI @@ -758,6 +758,20 @@ export class MarketClient { } } + /** + * Finalize a serial (single PUT) upload. + * @param id - Deal identifier (ULID string) + * @param deal - Optional deal payload to finalize with + */ + async finalizeSerialUpload(id: string, deal?: Mk20Deal): Promise { + try { + const result = await this.api.uploadIdPost({ id, body: deal }); + return result; + } catch (error) { + throw new Error(`Failed to finalize serial upload for deal ${id}: ${error}`); + } + } + /** * Get upload status for a deal * @param id - Deal identifier @@ -774,6 +788,20 @@ export class MarketClient { } } + /** + * Update an existing deal (e.g., request deletion via PDPv1 flags). + * @param id - Deal identifier (ULID string) + * @param deal - Deal payload with updated products + */ + async updateDeal(id: string, deal: Mk20Deal): Promise { + try { + const result = await this.api.updateIdGet({ id, body: deal }); + return result; + } catch (error) { + throw new Error(`Failed to update deal ${id}: ${error}`); + } + } + /** * Get info (placeholder method for compatibility) */ diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts index 2109e7dcd..b6d1ff42d 100644 --- a/market/mk20/tsclient/src/streaming.ts +++ b/market/mk20/tsclient/src/streaming.ts @@ -141,7 +141,7 @@ export class StreamingPDP { createDataSet: true, addPiece: true, recordKeeper: this.providerAddr, - extraData: [], + extraData: '', pieceIds: undefined, deleteDataSet: false, deletePiece: false, @@ -154,7 +154,7 @@ export class StreamingPDP { } as Products; const deal: Deal = { - identifier: this.identifierBytes, + identifier: this.id, client: this.clientAddr, products, } as Deal; @@ -216,7 +216,7 @@ export class StreamingPDP { const pieceCid = StreamingCommP.pieceCidV2FromDigest(this.totalSize, digest); const dataSource: Mk20DataSource = { - pieceCid: pieceCid as unknown as object, + pieceCid: pieceCid, format: { raw: {} } as Mk20PieceDataFormat, sourceHttpput: { raw_size: this.totalSize } as unknown as object, }; diff --git a/market/mk20/tsclient/tests/client.test.ts b/market/mk20/tsclient/tests/client.test.ts index 1f31bc250..db7085e30 100644 --- a/market/mk20/tsclient/tests/client.test.ts +++ b/market/mk20/tsclient/tests/client.test.ts @@ -88,7 +88,7 @@ describe('MarketClient', () => { describe('submitDeal', () => { it('should submit deal successfully', async () => { - const mockDeal = { identifier: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] }; + const mockDeal = { identifier: '01H0EXAMPLEULIDIDENTIFIER00000000' }; const mockResult = 200; // DealCode.Ok mockApi.storePost.mockResolvedValue(mockResult); From 420778d3a6898cca91ff5c0eed005c51a6f76c7d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Sun, 7 Sep 2025 13:45:21 -0500 Subject: [PATCH 35/55] auth fixes --- .../tsclient/examples/unpkg-end-to-end.ts | 145 +++++++- market/mk20/tsclient/package-lock.json | 274 ++++++++++++++- market/mk20/tsclient/package.json | 3 + market/mk20/tsclient/src/auth.ts | 319 ++++++++++++++++++ market/mk20/tsclient/src/index.ts | 2 + 5 files changed, 722 insertions(+), 21 deletions(-) create mode 100644 market/mk20/tsclient/src/auth.ts diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end.ts b/market/mk20/tsclient/examples/unpkg-end-to-end.ts index 3ec6aa97e..faa43ed1b 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end.ts @@ -1,10 +1,12 @@ // Set before running: // PDP_URL=https://andyserver.thepianoexpress.com // PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet -// PDP_PROVIDER=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // provider wallet -// PDP_TOKEN=your-token-here +// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode +// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode +// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 +// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... // PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 -import { Client, MarketClientConfig, PieceCidUtils } from '../src'; +import { Client, MarketClientConfig, PieceCidUtils, AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from '../src'; async function downloadFromUnpkg(url: string): Promise { const res = await fetch(url); @@ -18,33 +20,148 @@ async function sleep(ms: number) { } async function run() { + if (process.env.PDP_INSECURE_TLS === '1') { + // Disable TLS verification (use only for debugging!) + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + console.warn('WARNING: PDP_INSECURE_TLS=1 set. TLS verification disabled.'); + } + const config: MarketClientConfig = { serverUrl: process.env.PDP_URL || 'http://localhost:8080', - headers: process.env.PDP_TOKEN ? { Authorization: `Bearer ${process.env.PDP_TOKEN}` } : undefined, } as MarketClientConfig; - const client = new (require('../src').Client)(config); - const clientAddr = process.env.PDP_CLIENT || 'f1client...'; - const providerAddr = process.env.PDP_PROVIDER || 'f1provider...'; + const providerAddr = process.env.PDP_PROVIDER || 't1000'; const contractAddress = process.env.PDP_CONTRACT || '0x0000000000000000000000000000000000000000'; + // Build Authorization header + const keyType = (process.env.PDP_KEY_TYPE || 'ed25519').toLowerCase(); + let authHeader: string; + if (keyType === 'ed25519') { + const pubB64 = process.env.PDP_PUBLIC_KEY_B64 || ''; + const privB64 = process.env.PDP_PRIVATE_KEY_B64 || ''; + if (!pubB64 || !privB64) throw new Error('PDP_PUBLIC_KEY_B64 and PDP_PRIVATE_KEY_B64 must be set for ed25519'); + const pub = Uint8Array.from(Buffer.from(pubB64, 'base64')); + const priv = Uint8Array.from(Buffer.from(privB64, 'base64')); + const signer = new Ed25519KeypairSigner(pub, priv); + authHeader = await AuthUtils.buildAuthHeader(signer, 'ed25519'); + } else if (keyType === 'secp256k1') { + // Derive pubKeyBase64 from Filecoin address bytes + const addrStr = clientAddr; + const { Secp256k1AddressSigner } = require('../src'); + const addrBytes = Secp256k1AddressSigner.addressBytesFromString(addrStr); + const pubB64 = Buffer.from(addrBytes).toString('base64'); + if (!pubB64) throw new Error('Unable to derive address bytes from PDP_CLIENT'); + + // Load secp256k1 private key from env (HEX preferred, else B64) + const privHex = process.env.PDP_SECP_PRIVATE_KEY_HEX || ''; + const privB64 = process.env.PDP_SECP_PRIVATE_KEY_B64 || ''; + let priv: Uint8Array | undefined; + if (privHex) { + const clean = privHex.startsWith('0x') ? privHex.slice(2) : privHex; + if (clean.length !== 64) throw new Error('PDP_SECP_PRIVATE_KEY_HEX must be 32-byte (64 hex chars)'); + const bytes = new Uint8Array(32); + for (let i = 0; i < 32; i++) bytes[i] = parseInt(clean.substr(i * 2, 2), 16); + priv = bytes; + } else if (privB64) { + const buf = Buffer.from(privB64, 'base64'); + if (buf.length !== 32) throw new Error('PDP_SECP_PRIVATE_KEY_B64 must decode to 32 bytes'); + priv = new Uint8Array(buf); + } + if (!priv) throw new Error('Set PDP_SECP_PRIVATE_KEY_HEX or PDP_SECP_PRIVATE_KEY_B64 for secp256k1 signing'); + + // Use Secp256k1AddressSigner (address bytes derived from PDP_CLIENT) + const signer = new Secp256k1AddressSigner(clientAddr, priv); + authHeader = await AuthUtils.buildAuthHeader(signer, 'secp256k1'); + } else { + throw new Error(`Unsupported PDP_KEY_TYPE: ${keyType}`); + } + + const client = new (require('../src').Client)({ + ...config, + headers: { Authorization: authHeader }, + } as MarketClientConfig); + + // Debug: show sanitized auth + const sanitize = (h: string) => h.replace(/:[A-Za-z0-9+/=]{16,}:/, (m) => `:${m.slice(1, 9)}...:`); + console.log('Auth header (sanitized):', sanitize(authHeader)); + console.log('Server URL:', config.serverUrl); + + // Debug: preflight connectivity + try { + const base = config.serverUrl.replace(/\/$/, ''); + const urls: Array<{ url: string; headers?: Record }> = [ + { url: `${base}/health` }, + { url: `${base}/market/mk20/info/swagger.json` }, + { url: `${base}/market/mk20/products`, headers: { Authorization: authHeader } }, + ]; + for (const { url, headers } of urls) { + try { + const init: RequestInit = headers ? { headers } : {}; + const r = await fetch(url, init); + console.log(`Preflight ${url}:`, r.status); + if (!r.ok) { + const text = await r.text().catch(() => ''); + console.log(`Preflight body (${url}):`, text); + } + } catch (e) { + const err = e as any; + console.error(`Preflight failed (${url}):`, err?.message || String(e), err?.cause?.code || '', err?.code || ''); + } + } + } catch (e) { + console.error('Preflight orchestrator failed:', (e as Error).message); + } + const targetUrl = 'https://unpkg.com/react@18/umd/react.production.min.js'; console.log(`ā¬‡ļø Downloading: ${targetUrl}`); const bytes = await downloadFromUnpkg(targetUrl); console.log(` Downloaded ${bytes.length} bytes`); // Compute piece CID locally for retrieval - const blob = new Blob([bytes], { type: 'application/octet-stream' }); + const blob = new Blob([Buffer.from(bytes)], { type: 'application/octet-stream' }); const pieceCid = await PieceCidUtils.computePieceCidV2([blob]); console.log(`šŸ”— Computed piece CID: ${pieceCid}`); console.log('šŸ“Ø Submitting PDPv1 deal and uploading via helper'); - const res = await client.submitPDPv1DealWithUpload({ - blobs: [blob], - client: clientAddr, - provider: providerAddr, - contractAddress, - }); + let res; + try { + res = await client.submitPDPv1DealWithUpload({ + blobs: [blob], + client: clientAddr, + provider: providerAddr, + contractAddress, + }); + } catch (e) { + console.error('Submit error:', (e as Error).message); + try { + const re: any = e as any; + if (re && re.response) { + const status = re.response.status; + const text = await re.response.text().catch(() => ''); + console.error('Submit error status:', status); + console.error('Submit error body:', text); + } + } catch (_) {} + // Extra debug: try store-only minimal deal to isolate + try { + const minimal = { + client: clientAddr, + products: { pdpV1: { createDataSet: true, addPiece: true, recordKeeper: providerAddr } }, + } as any; + const url = config.serverUrl.replace(/\/$/, '') + '/market/mk20/store'; + const r = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: authHeader }, + body: JSON.stringify(minimal), + }); + console.log('Direct /store status:', r.status); + const body = await r.text().catch(() => ''); + console.log('Direct /store body:', body); + } catch (ee) { + console.error('Direct /store failed:', (ee as Error).message); + } + throw e; + } const uploadId = res.uploadId; console.log('ā³ Polling deal status until complete/failed'); diff --git a/market/mk20/tsclient/package-lock.json b/market/mk20/tsclient/package-lock.json index 0a9d1573a..554377caf 100644 --- a/market/mk20/tsclient/package-lock.json +++ b/market/mk20/tsclient/package-lock.json @@ -9,8 +9,11 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@glif/filecoin-address": "^4.0.0", + "@noble/secp256k1": "^2.1.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", + "tweetnacl": "^1.0.3", "ulid": "^2.3.0" }, "devDependencies": { @@ -25,6 +28,12 @@ "node": ">=18.0.0" } }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.10.1.tgz", + "integrity": "sha512-96Z2IP3mYmF1Xg2cDm8f1gWGf/HUVedQ3FMifV4kG/PQ4yEP51xDtRAEfhVNt5f/uzpNkZHwWQuUcu6D6K+Ekw==", + "license": "MIT" + }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -546,6 +555,19 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@glif/filecoin-address": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@glif/filecoin-address/-/filecoin-address-4.0.0.tgz", + "integrity": "sha512-NgAM/EyPzRXKw3Uz331BjqIHH1nFfP9Gs52LyjUMcHhKrDrnp5WbY63yu+rGws9q9wAMl8jCjVD5VhN+AcUBqw==", + "license": "(Apache-2.0 OR MIT)", + "dependencies": { + "blakejs": "1.2.1", + "borc": "3.0.0", + "ethers": "6.13.2", + "leb128": "0.0.5", + "uint8arrays": "3.1.0" + } + }, "node_modules/@inquirer/external-editor": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.1.tgz", @@ -1183,6 +1205,39 @@ } } }, + "node_modules/@noble/curves": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz", + "integrity": "sha512-oYclrNgRaM9SsBUBVbb8M6DTV7ZHRTKugureoYEncY5c65HOmRzvSiTE3y5CYaPYJA/GVkrhXEoF0M3Ya9PMnw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.3.2" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.3.2.tgz", + "integrity": "sha512-MVC8EAQp7MvEcm30KWENFjgR+Mkmf+D189XJTkFIlwohU5hcBbn1ZkKq7KVTi2Hme3PMGF390DaL52beVrIihQ==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/secp256k1": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-2.3.0.tgz", + "integrity": "sha512-0TQed2gcBbIrh7Ccyw+y/uZQvbJwm7Ao4scBUxqpBCcsOlZG0O4KGfjtNAy/li4W8n1xt3dxrwJ0beZ2h2G6Kw==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nuxt/opencollective": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/@nuxt/opencollective/-/opencollective-0.4.1.tgz", @@ -1290,6 +1345,15 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@sovpro/delimited-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@sovpro/delimited-stream/-/delimited-stream-1.1.0.tgz", + "integrity": "sha512-kQpk267uxB19X3X2T1mvNMjyvIEonpNSHrMlK5ZaBU6aZxw7wPbpgKJOjHN3+/GPVpXgAV9soVT2oyHpLkLtyw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/@tokenizer/inflate": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz", @@ -1450,6 +1514,12 @@ "dev": true, "license": "MIT" }, + "node_modules/aes-js": { + "version": "4.0.0-beta.5", + "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-4.0.0-beta.5.tgz", + "integrity": "sha512-G965FqalsNyrPqgEGON7nIx1e/OVENSgiEIzyC63haUMuvNnwIgIjMs52hlTCKhkBny7A2ORNlfY9Zu+jmGk1Q==", + "license": "MIT" + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -1685,7 +1755,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -1712,6 +1781,15 @@ "node": ">=10.0.0" } }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -1724,6 +1802,72 @@ "readable-stream": "^3.4.0" } }, + "node_modules/blakejs": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/blakejs/-/blakejs-1.2.1.tgz", + "integrity": "sha512-QXUSXI3QVc/gJME0dBpXrag1kbzOqCjCX8/b54ntNyW6sjtoqxqRk3LTmXzaJoh71zMsDCjM+47jS7XiwN/+fQ==", + "license": "MIT" + }, + "node_modules/bn.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.2.tgz", + "integrity": "sha512-v2YAxEmKaBLahNwE1mjp4WON6huMNeuDvagFZW+ASCuA/ku0bXR9hSMw0XpiqMoA3+rmnyck/tPRSFQkoC9Cuw==", + "license": "MIT" + }, + "node_modules/borc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/borc/-/borc-3.0.0.tgz", + "integrity": "sha512-ec4JmVC46kE0+layfnwM3l15O70MlFiEbmQHY/vpqIKiUtPVntv4BY4NVnz3N4vb21edV3mY97XVckFvYHWF9g==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0", + "buffer": "^6.0.3", + "commander": "^2.15.0", + "ieee754": "^1.1.13", + "iso-url": "^1.1.5", + "json-text-sequence": "~0.3.0", + "readable-stream": "^3.6.0" + }, + "bin": { + "cbor2comment": "bin/cbor2comment.js", + "cbor2diag": "bin/cbor2diag.js", + "cbor2json": "bin/cbor2json.js", + "json2cbor": "bin/json2cbor.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/borc/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/borc/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -1836,6 +1980,15 @@ "dev": true, "license": "MIT" }, + "node_modules/buffer-pipe": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/buffer-pipe/-/buffer-pipe-0.0.3.tgz", + "integrity": "sha512-GlxfuD/NrKvCNs0Ut+7b1IHjylfdegMBxQIlZHj7bObKVQBxB5S84gtm2yu1mQ8/sSggceWBDPY0cPXgvX2MuA==", + "license": "MPL-2.0", + "dependencies": { + "safe-buffer": "^5.1.2" + } + }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -2515,6 +2668,46 @@ "node": ">=0.10.0" } }, + "node_modules/ethers": { + "version": "6.13.2", + "resolved": "https://registry.npmjs.org/ethers/-/ethers-6.13.2.tgz", + "integrity": "sha512-9VkriTTed+/27BGuY1s0hf441kqwHJ1wtN2edksEtiRvXx+soxRX3iSXTfFqq2+YwrOqbDoTHjIhQnjJRlzKmg==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/ethers-io/" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "1.10.1", + "@noble/curves": "1.2.0", + "@noble/hashes": "1.3.2", + "@types/node": "18.15.13", + "aes-js": "4.0.0-beta.5", + "tslib": "2.4.0", + "ws": "8.17.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/ethers/node_modules/@types/node": { + "version": "18.15.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", + "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==", + "license": "MIT" + }, + "node_modules/ethers/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "license": "0BSD" + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -3044,7 +3237,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -3107,7 +3299,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/inquirer": { @@ -3243,6 +3434,15 @@ "dev": true, "license": "ISC" }, + "node_modules/iso-url": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/iso-url/-/iso-url-1.2.1.tgz", + "integrity": "sha512-9JPDgCN4B7QPkLtYAAOrEuAWvP9rWvR5offAr0/SeF046wIkglqH3VXgYYP6NcsKslH80UIVgmPqNe3j7tG2ng==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/isomorphic-fetch": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", @@ -4070,6 +4270,18 @@ "dev": true, "license": "MIT" }, + "node_modules/json-text-sequence": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/json-text-sequence/-/json-text-sequence-0.3.0.tgz", + "integrity": "sha512-7khKIYPKwXQem4lWXfpIN/FEnhztCeRPSxH4qm3fVlqulwujrRDD54xAwDDn/qVKpFtV550+QAkcWJcufzqQuA==", + "license": "MIT", + "dependencies": { + "@sovpro/delimited-stream": "^1.1.0" + }, + "engines": { + "node": ">=10.18.0" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -4106,6 +4318,16 @@ "node": ">=6" } }, + "node_modules/leb128": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/leb128/-/leb128-0.0.5.tgz", + "integrity": "sha512-elbNtfmu3GndZbesVF6+iQAfVjOXW9bM/aax9WwMlABZW+oK9sbAZEXoewaPHmL34sxa8kVwWsru8cNE/yn2gg==", + "license": "MPL-2.0", + "dependencies": { + "bn.js": "^5.0.0", + "buffer-pipe": "0.0.3" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -4833,7 +5055,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -4960,7 +5181,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -5156,7 +5376,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -5508,6 +5727,12 @@ "dev": true, "license": "0BSD" }, + "node_modules/tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", + "license": "Unlicense" + }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -5585,6 +5810,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/uint8arrays": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-3.1.0.tgz", + "integrity": "sha512-ei5rfKtoRO8OyOIor2Rz5fhzjThwIHJZ3uyDPnDHTXbP0aMQ1RN/6AI5B5d9dBxJOU+BvOAk7ZQ1xphsX8Lrog==", + "license": "MIT", + "dependencies": { + "multiformats": "^9.4.2" + } + }, + "node_modules/uint8arrays/node_modules/multiformats": { + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", + "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", + "license": "(Apache-2.0 AND MIT)" + }, "node_modules/ulid": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.4.0.tgz", @@ -5646,7 +5886,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/v8-to-istanbul": { @@ -5791,6 +6030,27 @@ "dev": true, "license": "ISC" }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 7789b639c..9822642d8 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -33,8 +33,11 @@ "typescript": "^5.0.0" }, "dependencies": { + "@glif/filecoin-address": "^4.0.0", + "@noble/secp256k1": "^2.1.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", + "tweetnacl": "^1.0.3", "ulid": "^2.3.0" }, "engines": { diff --git a/market/mk20/tsclient/src/auth.ts b/market/mk20/tsclient/src/auth.ts new file mode 100644 index 000000000..99fb6ab2c --- /dev/null +++ b/market/mk20/tsclient/src/auth.ts @@ -0,0 +1,319 @@ +// Lazy import to avoid hard dependency during build environments without install +let nacl: any; +async function getNacl(): Promise { + if (!nacl) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + nacl = require('tweetnacl'); + } + return nacl; +} + +let nobleSecp: any; +async function getSecp(): Promise { + if (!nobleSecp) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + nobleSecp = require('@noble/secp256k1'); + // Provide sync HMAC-SHA256 for RFC6979 (required by noble's sign) + try { + if (!nobleSecp.etc.hmacSha256Sync) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const nodeCrypto = require('crypto'); + const concat = (...arrs: Uint8Array[]) => { + const total = arrs.reduce((s, a) => s + a.length, 0); + const out = new Uint8Array(total); + let off = 0; + for (const a of arrs) { out.set(a, off); off += a.length; } + return out; + }; + nobleSecp.etc.hmacSha256Sync = (key: Uint8Array, ...msgs: Uint8Array[]) => { + const h = nodeCrypto.createHmac('sha256', Buffer.from(key)); + const all = concat(...msgs); + h.update(Buffer.from(all)); + return new Uint8Array(h.digest()); + }; + } + } catch (_) { + // leave as-is; if not set, noble will throw, which surfaces clearly + } + } + return nobleSecp; +} + +/** + * Utilities to construct Curio Market 2.0 Authorization headers. + * + * Header format: + * Authorization: "CurioAuth ::" + * + * - For ed25519: + * - pubKeyBase64: base64 of 32-byte raw public key + * - signatureBase64: base64 of detached ed25519 signature over sha256(pubKey || RFC3339Hour) + * - For secp256k1 / bls / delegated: not implemented here (Filecoin signature envelope required) + */ +export class AuthUtils { + /** Signer interface for pluggable key types. */ + static readonly KEYTYPE_ED25519 = 'ed25519' as const; + + /** + * Build Authorization header from a provided signer and key type. + * Currently supports 'ed25519'. + */ + static async buildAuthHeader( + signer: AuthSigner, + keyType: 'ed25519' | 'secp256k1', + now?: Date, + ): Promise { + switch (keyType) { + case 'ed25519': + return this.buildEd25519AuthHeader(await signer.getPublicKey(), await signer.sign.bind(signer), now); + case 'secp256k1': { + const addrBytes = await signer.getPublicKey(); + const ts = this.rfc3339TruncatedToHour(now); + const msg = await this.sha256Concat(addrBytes, new TextEncoder().encode(ts)); + const sigEnvelope = await signer.sign(msg); // expected to be Filecoin signature envelope bytes + const pubB64 = this.toBase64(addrBytes); + const sigB64 = this.toBase64(sigEnvelope); + return `CurioAuth secp256k1:${pubB64}:${sigB64}`; + } + default: + throw new Error(`Unsupported key type: ${keyType}`); + } + } + + /** + * Build Authorization header for ed25519 keys. + * @param publicKeyRaw - 32-byte ed25519 public key (raw) + * @param privateKeyOrSign - ed25519 private key bytes (64 secretKey or 32 seed), + * OR a sign function (message)=>signature + * @param now - Optional date used for timestamp; defaults to current time + * @returns Authorization header value (without the "Authorization: " prefix) + */ + static async buildEd25519AuthHeader( + publicKeyRaw: Uint8Array, + privateKeyOrSign: Uint8Array | ((message: Uint8Array) => Promise | Uint8Array), + now?: Date, + ): Promise { + if (publicKeyRaw.length !== 32) { + throw new Error(`ed25519 publicKey must be 32 bytes, got ${publicKeyRaw.length}`); + } + + const ts = this.rfc3339TruncatedToHour(now); + const message = await this.sha256Concat(publicKeyRaw, new TextEncoder().encode(ts)); + let signature: Uint8Array; + if (typeof privateKeyOrSign === 'function') { + signature = await privateKeyOrSign(message); + } else { + const secretKey = this.ensureEd25519SecretKey(privateKeyOrSign); + const n = await getNacl(); + signature = n.sign.detached(message, secretKey); + } + + const pubB64 = this.toBase64(publicKeyRaw); + const sigB64 = this.toBase64(signature); + return `CurioAuth ed25519:${pubB64}:${sigB64}`; + } + + /** Return headers object with Authorization set for ed25519. */ + static async makeAuthHeadersEd25519( + publicKeyRaw: Uint8Array, + privateKey: Uint8Array, + now?: Date, + ): Promise> { + const value = await this.buildEd25519AuthHeader(publicKeyRaw, privateKey, now); + return { Authorization: value }; + } + + /** Convert a 32-byte seed or 64-byte secretKey into a 64-byte secretKey. */ + private static ensureEd25519SecretKey(privateKey: Uint8Array): Uint8Array { + if (privateKey.length === 64) { + return privateKey; + } + if (privateKey.length === 32) { + const n = require('tweetnacl'); + const kp = n.sign.keyPair.fromSeed(privateKey); + return kp.secretKey; + } + throw new Error(`ed25519 private key must be 32-byte seed or 64-byte secretKey, got ${privateKey.length}`); + } + + /** RFC3339 timestamp truncated to the hour, always UTC, e.g., 2025-07-15T17:00:00Z */ + static rfc3339TruncatedToHour(date?: Date): string { + const d = date ? new Date(date) : new Date(); + const y = d.getUTCFullYear(); + const m = (d.getUTCMonth() + 1).toString().padStart(2, '0'); + const day = d.getUTCDate().toString().padStart(2, '0'); + const h = d.getUTCHours().toString().padStart(2, '0'); + return `${y}-${m}-${day}T${h}:00:00Z`; + } + + /** Compute sha256 over concatenation of two byte arrays. */ + private static async sha256Concat(a: Uint8Array, b: Uint8Array): Promise { + const combined = new Uint8Array(a.length + b.length); + combined.set(a, 0); + combined.set(b, a.length); + // Prefer WebCrypto when available + if (typeof globalThis !== 'undefined' && (globalThis as any).crypto?.subtle) { + const hashBuf = await (globalThis as any).crypto.subtle.digest('SHA-256', combined); + return new Uint8Array(hashBuf); + } + // Fallback to Node crypto + try { + const nodeCrypto = await import('crypto'); + const hasher = nodeCrypto.createHash('sha256'); + hasher.update(Buffer.from(combined)); + return new Uint8Array(hasher.digest()); + } catch { + throw new Error('No available crypto implementation to compute SHA-256 digest'); + } + } + + /** Base64 encode Uint8Array across environments. */ + private static toBase64(bytes: Uint8Array): string { + if (typeof Buffer !== 'undefined') { + // Node + return Buffer.from(bytes).toString('base64'); + } + // Browser + let binary = ''; + for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); + return btoa(binary); + } + + /** Compute BLAKE2b-256 digest (32 bytes). */ + static async blake2b256(data: Uint8Array): Promise { + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const nodeCrypto = require('crypto'); + try { + const h = nodeCrypto.createHash('blake2b512', { outputLength: 32 }); + h.update(Buffer.from(data)); + return new Uint8Array(h.digest()); + } catch (_) { + // fall back to blakejs + } + } catch (_) {} + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const blake = require('blakejs'); + const out = blake.blake2b(data, undefined, 32); + return new Uint8Array(out); + } catch (_) { + throw new Error('No available BLAKE2b-256 implementation'); + } + } +} + +export default AuthUtils; + +/** Generic signer interface */ +export interface AuthSigner { + getPublicKey(): Promise | Uint8Array; + sign(message: Uint8Array): Promise | Uint8Array; +} + +/** Ed25519 signer that takes public and private key material at construction */ +export class Ed25519KeypairSigner implements AuthSigner { + private readonly publicKeyRaw: Uint8Array; + private readonly secretKey: Uint8Array; + + constructor(publicKeyRaw: Uint8Array, privateKey: Uint8Array) { + if (publicKeyRaw.length !== 32) { + throw new Error(`ed25519 publicKey must be 32 bytes, got ${publicKeyRaw.length}`); + } + this.publicKeyRaw = publicKeyRaw; + this.secretKey = AuthUtils['ensureEd25519SecretKey'](privateKey); + } + + getPublicKey(): Uint8Array { + return this.publicKeyRaw; + } + + async sign(message: Uint8Array): Promise { + const n = await getNacl(); + return n.sign.detached(message, this.secretKey); + } +} + +/** Secp256k1 signer using a Filecoin address and secp256k1 private key. */ +export class Secp256k1AddressSigner implements AuthSigner { + private readonly addressBytes: Uint8Array; + private readonly privateKey: Uint8Array; + + /** + * @param addressString - Filecoin address string (f1/t1) + * @param privateKey - 32-byte secp256k1 private key (Uint8Array) + */ + constructor(addressString: string, privateKey: Uint8Array) { + this.addressBytes = Secp256k1AddressSigner.addressBytesFromString(addressString); + if (privateKey.length !== 32) { + throw new Error(`secp256k1 private key must be 32 bytes, got ${privateKey.length}`); + } + this.privateKey = privateKey; + } + + getPublicKey(): Uint8Array { + // For secp256k1 CurioAuth, the "public key" field is the Filecoin address bytes + return this.addressBytes; + } + + /** + * Produce Filecoin signature envelope bytes: [SigType=0x01] || [65-byte secp256k1 signature (R||S||V)] + */ + async sign(message: Uint8Array): Promise { + const secp = await getSecp(); + const digest = await AuthUtils.blake2b256(message); + const sigObj = secp.sign(digest, this.privateKey); // returns Signature with recovery + const sig = typeof sigObj.toCompactRawBytes === 'function' ? sigObj.toCompactRawBytes() : sigObj.toBytes(); + const recid = sigObj.recovery ?? 0; + if (!(sig instanceof Uint8Array) || sig.length !== 64) throw new Error('unexpected secp256k1 signature size'); + const data = new Uint8Array(1 + 65); + data[0] = 0x01; // fcrypto.SigTypeSecp256k1 + data.set(sig, 1); + data[1 + 64] = recid & 0xff; + return data; + } + + /** Parse Filecoin f1/t1 address string to address bytes: [protocol (1)] || payload (20). */ + static addressBytesFromString(address: string): Uint8Array { + if (!address || address.length < 3) throw new Error('invalid address'); + const net = address[0]; + if (net !== 'f' && net !== 't') throw new Error('invalid network prefix'); + const protoCh = address[1]; + if (protoCh !== '1') throw new Error('unsupported protocol: only secp256k1 (1) supported'); + const b32 = address.slice(2).toLowerCase(); + const decoded = Secp256k1AddressSigner.base32Decode(b32); + if (decoded.length < 4 + 20) throw new Error('invalid address payload'); + const payload = decoded.slice(0, decoded.length - 4); // drop checksum (last 4 bytes) + if (payload.length !== 20) throw new Error('invalid secp256k1 payload length'); + const out = new Uint8Array(1 + payload.length); + out[0] = 0x01; // protocol 1 + out.set(payload, 1); + return out; + } + + /** Base32 decode with alphabet 'abcdefghijklmnopqrstuvwxyz234567'. */ + private static base32Decode(s: string): Uint8Array { + const alphabet = 'abcdefghijklmnopqrstuvwxyz234567'; + const map: Record = {}; + for (let i = 0; i < alphabet.length; i++) map[alphabet[i]] = i; + let bits = 0; + let value = 0; + const out: number[] = []; + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + if (ch === '=') break; + const v = map[ch]; + if (v === undefined) throw new Error('invalid base32 character'); + value = (value << 5) | v; + bits += 5; + if (bits >= 8) { + out.push((value >> (bits - 8)) & 0xff); + bits -= 8; + value &= (1 << bits) - 1; + } + } + return new Uint8Array(out); + } +} + + diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts index 37af1acf9..7347ffdd6 100644 --- a/market/mk20/tsclient/src/index.ts +++ b/market/mk20/tsclient/src/index.ts @@ -26,6 +26,8 @@ export type { MarketClientConfig } from './client'; // Export piece CID utilities export { PieceCidUtils } from './client'; export { StreamingPDP } from './streaming'; +export { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; +export type { AuthSigner } from './auth'; // Re-export configuration types export type { Configuration } from '../generated'; From 32a9ebaaa6fbb348f08ed8a007ac0379a01f3d58 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 8 Sep 2025 11:21:19 -0500 Subject: [PATCH 36/55] added wait, sep deal calls --- market/mk20/tsclient/src/client.ts | 179 +++++++---------------------- 1 file changed, 39 insertions(+), 140 deletions(-) diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index e4a60c992..7afd659b4 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -1,4 +1,4 @@ -import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources, Mk20Products, Mk20PDPV1, Mk20RetrievalV1, Mk20DDOV1, Mk20DataSource } from '../generated'; +import { DefaultApi, ConfigurationParameters, Mk20Deal, Mk20DealProductStatusResponse, Mk20SupportedContracts, Mk20SupportedProducts, Mk20SupportedDataSources, Mk20Products, Mk20PDPV1, Mk20RetrievalV1, Mk20DDOV1, Mk20DataSource, Mk20DealState } from '../generated'; import { ulid } from 'ulid'; import { Configuration } from '../generated/runtime'; import { Mk20StartUpload } from '../generated/models/Mk20StartUpload'; @@ -386,7 +386,6 @@ export class MarketClient { } - /** * Calculate piece ID for an individual blob based on its content * @param blob - The blob to calculate piece ID for @@ -411,6 +410,22 @@ export class MarketClient { return Math.abs(hash) % 1000000; // Keep within 6 digits } + async waitDealComplete(id: string): Promise { + var duration = 0; + const step = 10000; + while (true) { + const resp = await this.getStatus(id); + if (resp?.pdpV1?.status === Mk20DealState.DealStateComplete) { + break + } + + await new Promise(resolve => setTimeout(resolve, step)); + duration += step; + if (duration > 90000) { + throw new Error(`Deal ${id} timed out after ${duration} seconds`); + } + } + } /** * Simple convenience wrapper for PDPv1 deals with chunked upload * Takes blobs and required addresses, computes piece_cid, and returns a UUID identifier @@ -450,28 +465,20 @@ export class MarketClient { // Compute piece_cid from blobs using our utility (WebCrypto SubtleCrypto) const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); - // Create deal with required addresses var deal: Mk20Deal = { // Use the generated UUID as the deal identifier identifier: uuid, client, - data: { - pieceCid: pieceCid, - format: { raw: {} }, - sourceHttpput: { - raw_size: totalSize - } as unknown as object, - } as Mk20DataSource, products: { pdpV1: { createDataSet: true, // Create a new dataset for this deal - addPiece: true, // Add the piece to the dataset - dataSetId: undefined, // Not needed when creating dataset + //addPiece: true, // Add the piece to the dataset + //dataSetId: undefined, // Not needed when creating dataset recordKeeper: provider, // Use provider as record keeper extraData: '', // No extra data - pieceIds: undefined, // Piece IDs (on chain) not available for new content. + //pieceIds: undefined, // Piece IDs (on chain) not available for new content. deleteDataSet: false, deletePiece: false } as Mk20PDPV1, @@ -484,137 +491,30 @@ export class MarketClient { }; // Submit the deal - const dealId = await this.submitDeal(deal); - - // Initialize chunked upload - const startUpload: Mk20StartUpload = { - rawSize: totalSize, - chunkSize: 1024 * 1024 // 1MB chunks - }; + var dealId = await this.submitDeal(deal); - const uploadInitResult = await this.initializeChunkedUpload(uuid, startUpload); + this.waitDealComplete(uuid); - // Automatically upload all blobs in chunks - console.log(`šŸ“¤ Starting automatic chunked upload of ${blobs.length} blobs...`); - const chunkSize = 1024 * 1024; // 1MB chunks - let totalChunks = 0; - let uploadedBytes = 0; - - for (const [blobIndex, blob] of blobs.entries()) { - const blobSize = blob.size; - const blobChunks = Math.ceil(blobSize / chunkSize); - - console.log(` Uploading blob ${blobIndex + 1}/${blobs.length} (${blobSize} bytes, ${blobChunks} chunks)...`); - - for (let i = 0; i < blobSize; i += chunkSize) { - const chunk = blob.slice(i, i + chunkSize); - const chunkNum = totalChunks.toString(); - - // Convert blob chunk to array of numbers for upload - const chunkArray = new Uint8Array(await chunk.arrayBuffer()); - const chunkNumbers = Array.from(chunkArray); - - console.log(` Uploading chunk ${chunkNum + 1} (${chunkNumbers.length} bytes)...`); - await this.uploadChunk(uuid, chunkNum, chunkNumbers); - - totalChunks++; - uploadedBytes += chunkNumbers.length; - } - } - - // Finalize the upload - console.log('šŸ”’ Finalizing chunked upload...'); - const finalizeResult = await this.finalizeChunkedUpload(uuid, deal); - console.log(`āœ… Upload finalized: ${finalizeResult}`); - - return { - uuid, - totalSize, - dealId, - uploadId: uuid, - pieceCid, - uploadedChunks: totalChunks, - uploadedBytes - }; - - } catch (error) { - throw new Error(`Failed to submit PDPv1 deal with upload: ${error}`); - } - } - - /** - * Simple convenience wrapper for DDO deals with chunked upload - * Takes blobs and required addresses, computes piece_cid, and returns a UUID identifier - */ - /** - * Convenience wrapper for DDOv1 deals with chunked upload. - * @param params - Input parameters - * @param params.blobs - Data to upload as an array of blobs - * @param params.client - Client wallet address - * @param params.provider - Provider wallet address - * @param params.contractAddress - Verification contract address - * @param params.lifespan - Optional deal lifespan in epochs (defaults to 518400) - * @returns Upload metadata including uuid, pieceCid, and stats - */ - async submitDDOV1DealWithUpload(params: { - blobs: Blob[]; - client: string; - provider: string; - contractAddress: string; - lifespan?: number; - }): Promise<{ - uuid: string; - totalSize: number; - dealId: number; - uploadId: string; - pieceCid: string; - pieceIds: number[]; - uploadedChunks: number; - uploadedBytes: number; - }> { - try { - const { blobs, client, provider, contractAddress } = params; - const duration = params.lifespan ?? 518400; - - // Calculate total size from blobs - const totalSize = blobs.reduce((sum, blob) => sum + blob.size, 0); - - // Generate a ULID for the deal identifier returned to the caller - const uuid = ulid(); - - // Compute piece_cid from blobs using our utility (WebCrypto SubtleCrypto) - const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); - - // Calculate piece IDs for each individual blob - const pieceIds: number[] = []; - for (const blob of blobs) { - const pieceId = await this.calculateBlobPieceId(blob); - pieceIds.push(pieceId); - } - // Create deal with required addresses - const deal: Mk20Deal = { + var deal: Mk20Deal = { // Use the generated UUID as the deal identifier identifier: uuid, client, data: { pieceCid: pieceCid, format: { raw: {} }, - sourceHttpput: { - raw_size: totalSize - } as unknown as object, - } as any, + sourceHttpput: {}, + } as Mk20DataSource, products: { - ddoV1: { - duration, // Deal duration in epochs - provider: { address: provider }, - contractAddress, - contractVerifyMethod: 'verifyDeal', - contractVerifyMethodParams: '', - pieceManager: { address: provider }, - notificationAddress: client, - notificationPayload: '' - } as Mk20DDOV1, + pdpV1: { + addPiece: true, // Add the piece to the dataset + //dataSetId: undefined, // Not needed when creating dataset + recordKeeper: provider, // Use provider as record keeper + extraData: '', // No extra data + //pieceIds: undefined, // Piece IDs (on chain) not available for new content. + deleteDataSet: false, + deletePiece: false + } as Mk20PDPV1, retrievalV1: { announcePayload: true, // Announce payload to IPNI announcePiece: true, // Announce piece information to IPNI @@ -623,8 +523,9 @@ export class MarketClient { } as Mk20Products }; - // Submit the deal - const dealId = await this.submitDeal(deal); + var dealId = await this.submitDeal(deal); + + this.waitDealComplete(uuid); // Initialize chunked upload const startUpload: Mk20StartUpload = { @@ -664,7 +565,7 @@ export class MarketClient { // Finalize the upload console.log('šŸ”’ Finalizing chunked upload...'); - const finalizeResult = await this.finalizeChunkedUpload(uuid); + const finalizeResult = await this.finalizeChunkedUpload(uuid, deal); // TODO check deal (2nd ID) status in loop. console.log(`āœ… Upload finalized: ${finalizeResult}`); return { @@ -673,16 +574,14 @@ export class MarketClient { dealId, uploadId: uuid, pieceCid, - pieceIds, uploadedChunks: totalChunks, uploadedBytes }; } catch (error) { - throw new Error(`Failed to submit DDOv1 deal with upload: ${error}`); + throw new Error(`Failed to submit PDPv1 deal with upload: ${error}`); } } - /** * Upload deal data */ From b05d9c9f6de3877dbee1e5e173ee8eb4b6644974 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 9 Sep 2025 00:20:38 +0400 Subject: [PATCH 37/55] switch to camelCase for json --- cmd/curio/cli.go | 5 +- cmd/curio/debug-ipni.go | 6 +- cmd/curio/debug-snsvc.go | 2 +- cmd/curio/guidedsetup/guidedsetup.go | 6 +- cmd/curio/market.go | 4 +- cmd/curio/rpc/rpc.go | 2 +- cmd/curio/storage.go | 4 +- cmd/curio/tasks/tasks.go | 6 +- cmd/curio/toolbox.go | 4 +- cmd/curio/unseal.go | 4 +- cmd/pdptool/main.go | 77 +- cmd/sptool/evm.go | 6 +- cmd/sptool/toolbox_deal_client.go | 38 +- cmd/sptool/toolbox_deal_tools.go | 4 +- cuhttp/server.go | 9 +- deps/apiinfo.go | 2 +- deps/config/load.go | 2 +- deps/config/old_lotus_miner.go | 4 +- .../harmonydb/sql/20250505-market-mk20.sql | 143 +++- harmony/harmonydb/userfuncs.go | 2 +- harmony/harmonytask/harmonytask.go | 26 +- harmony/harmonytask/task_type_handler.go | 8 +- itests/pdp_prove_test.go | 5 +- lib/cachedreader/cachedreader.go | 4 +- lib/cachedreader/prefetch_test.go | 32 +- lib/chainsched/chain_sched_test.go | 10 +- lib/fastparamfetch/paramfetch.go | 12 +- lib/ffi/piece_funcs.go | 15 +- lib/ffi/scrub_funcs.go | 4 +- lib/ffi/unseal_funcs.go | 4 +- lib/hugepageutil/checkhuge.go | 4 +- lib/panicreport/panic_reporter.go | 6 +- lib/paths/db_index.go | 4 +- lib/paths/http_handler.go | 4 +- lib/paths/local.go | 2 +- lib/proofsvc/clientctl.go | 24 +- lib/proofsvc/provictl.go | 20 +- lib/testutils/testutils.go | 20 +- market/indexstore/indexstore.go | 1 - market/ipni/chunker/initial-chunker.go | 8 +- market/libp2p/redirector.go | 11 +- market/mk12/mk12.go | 11 +- market/mk20/client/http_client.go | 4 +- market/mk20/ddo_v1.go | 18 +- market/mk20/http/docs.go | 66 +- market/mk20/http/http.go | 28 +- market/mk20/http/swagger.json | 66 +- market/mk20/http/swagger.yaml | 66 +- market/mk20/mk20.go | 172 +++-- market/mk20/mk20_upload.go | 29 +- market/mk20/pdp_v1.go | 16 +- market/mk20/retrieval_v1.go | 4 +- market/mk20/types.go | 16 +- market/mk20/types_test.go | 297 ++++++++ market/mk20/utils.go | 16 +- market/retrieval/piecehandler.go | 2 +- market/storageingest/deal_ingest_seal.go | 2 +- market/storageingest/deal_ingest_snap.go | 2 +- .../ListenerServiceWithViewContract.abi | 15 + .../ListenerServiceWithViewContract.go | 212 ++++++ pdp/contract/addresses.go | 2 +- pdp/contract/utils.go | 37 + pdp/handlers.go | 12 +- pdp/handlers_upload.go | 9 +- tasks/gc/storage_gc_mark.go | 4 +- tasks/indexing/task_indexing.go | 4 +- tasks/indexing/task_ipni.go | 4 +- tasks/indexing/task_pdp_indexing.go | 4 +- tasks/indexing/task_pdp_ipni.go | 6 +- tasks/metadata/task_sector_expirations.go | 4 +- tasks/pdp/data_set_create_watch.go | 6 +- tasks/pdp/notify_task.go | 4 +- tasks/pdp/task_aggregation.go | 10 +- tasks/pdp/task_commp.go | 326 ++++++++ tasks/pdp/task_init_pp.go | 6 +- tasks/pdp/task_next_pp.go | 6 +- tasks/pdp/task_prove.go | 8 +- tasks/pdp/task_save_cache.go | 10 +- tasks/piece/task_aggregate_chunks.go | 8 +- tasks/sealsupra/supra_config.go | 2 +- tasks/storage-market/mk20.go | 176 +++-- tasks/storage-market/storage_market.go | 20 +- tasks/storage-market/task_aggregation.go | 4 +- tasks/winning/winning_task.go | 3 +- web/api/webrpc/market.go | 66 +- web/api/webrpc/{market_20.go => market_2.go} | 719 +++++++++++++++++- web/api/webrpc/proofshare.go | 4 +- web/srv.go | 8 +- web/static/pages/mk20-deal/deal.mjs | 22 +- web/static/pages/mk20/ddo-pipeline.mjs | 2 +- web/static/pages/mk20/settings.mjs | 17 +- web/static/pages/pdp/pipeline.mjs | 374 +++++++++ web/static/pages/piece/piece-info.mjs | 146 +++- 93 files changed, 3024 insertions(+), 605 deletions(-) create mode 100644 market/mk20/types_test.go create mode 100644 pdp/contract/ListenerServiceWithViewContract.abi create mode 100644 pdp/contract/ListenerServiceWithViewContract.go create mode 100644 pdp/contract/utils.go create mode 100644 tasks/pdp/task_commp.go rename web/api/webrpc/{market_20.go => market_2.go} (50%) create mode 100644 web/static/pages/pdp/pipeline.mjs diff --git a/cmd/curio/cli.go b/cmd/curio/cli.go index 7d6e57ca4..defdc50c4 100644 --- a/cmd/curio/cli.go +++ b/cmd/curio/cli.go @@ -229,10 +229,7 @@ var waitApiCmd = &cli.Command{ ctx := reqcontext.ReqContext(cctx) ctx, cancel := context.WithTimeout(ctx, cctx.Duration("timeout")) defer cancel() - for { - if ctx.Err() != nil { - break - } + for ctx.Err() == nil { api, closer, err := rpc.GetCurioAPI(cctx) if err != nil { diff --git a/cmd/curio/debug-ipni.go b/cmd/curio/debug-ipni.go index 6cd6559b2..5fc9de7d0 100644 --- a/cmd/curio/debug-ipni.go +++ b/cmd/curio/debug-ipni.go @@ -26,7 +26,9 @@ var testDebugIpniChunks = &cli.Command{ if err != nil { return xerrors.Errorf("opening file: %w", err) } - defer f.Close() + defer func() { + _ = f.Close() + }() opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} blockReader, err := carv2.NewBlockReader(bufio.NewReaderSize(f, 4<<20), opts...) @@ -36,7 +38,7 @@ var testDebugIpniChunks = &cli.Command{ blockMetadata, err := blockReader.SkipNext() for err == nil { - if err := ck.Accept(blockMetadata.Cid.Hash(), int64(blockMetadata.Offset), blockMetadata.Size+40); err != nil { + if err := ck.Accept(blockMetadata.Hash(), int64(blockMetadata.Offset), blockMetadata.Size+40); err != nil { return xerrors.Errorf("accepting block: %w", err) } diff --git a/cmd/curio/debug-snsvc.go b/cmd/curio/debug-snsvc.go index 5566f0707..fb1143c7f 100644 --- a/cmd/curio/debug-snsvc.go +++ b/cmd/curio/debug-snsvc.go @@ -590,7 +590,7 @@ func getClientStateAction(cctx *cli.Context) error { fmt.Printf("VoucherRedeemed: %s\n", types.FIL(clientState.VoucherRedeemed).String()) fmt.Printf("LastNonce: %d\n", clientState.LastNonce) fmt.Printf("WithdrawAmount: %s\n", types.FIL(clientState.WithdrawAmount).String()) - ts := clientState.WithdrawTimestamp.Int.Uint64() + ts := clientState.WithdrawTimestamp.Uint64() wt := time.Unix(int64(ts), 0) diff := time.Until(wt) var diffStr string diff --git a/cmd/curio/guidedsetup/guidedsetup.go b/cmd/curio/guidedsetup/guidedsetup.go index d29018156..76dd06729 100644 --- a/cmd/curio/guidedsetup/guidedsetup.go +++ b/cmd/curio/guidedsetup/guidedsetup.go @@ -37,7 +37,6 @@ import ( "github.com/filecoin-project/curio/api" "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/cmd/curio/internal/translations" - _ "github.com/filecoin-project/curio/cmd/curio/internal/translations" "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" @@ -163,9 +162,10 @@ func newOrMigrate(d *MigrationData) { d.say(notice, "Aborting remaining steps.", err.Error()) os.Exit(1) } - if i == 1 { + switch i { + case 1: d.init = true - } else if i == 2 { + case 2: d.nonSP = true } } diff --git a/cmd/curio/market.go b/cmd/curio/market.go index 745cd9f0e..318b107a3 100644 --- a/cmd/curio/market.go +++ b/cmd/curio/market.go @@ -130,7 +130,9 @@ var marketAddOfflineURLCmd = &cli.Command{ if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() diff --git a/cmd/curio/rpc/rpc.go b/cmd/curio/rpc/rpc.go index 818144a2f..a6647aea2 100644 --- a/cmd/curio/rpc/rpc.go +++ b/cmd/curio/rpc/rpc.go @@ -219,7 +219,7 @@ func (p *CurioAPI) StorageInit(ctx context.Context, path string, opts storiface. if opts.ID == "" { opts.ID = storiface.ID(uuid.New().String()) } - if !(opts.CanStore || opts.CanSeal) { + if !opts.CanStore && !opts.CanSeal { return xerrors.Errorf("must specify at least one of --store or --seal") } b, err := json.MarshalIndent(opts, "", " ") diff --git a/cmd/curio/storage.go b/cmd/curio/storage.go index 0bd1130ac..393b03074 100644 --- a/cmd/curio/storage.go +++ b/cmd/curio/storage.go @@ -188,7 +188,7 @@ over time DenyMiners: cctx.StringSlice("deny-miners"), } - if !(cfg.CanStore || cfg.CanSeal) { + if !cfg.CanStore && !cfg.CanSeal { return xerrors.Errorf("must specify at least one of --store or --seal") } @@ -579,7 +579,7 @@ var storageGenerateVanillaProofCmd = &cli.Command{ if err != nil { return xerrors.Errorf("generating proof: %w", err) } - fmt.Println(proof) + fmt.Println(string(proof)) return nil }, } diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index e4578f82f..530e61f04 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -318,8 +318,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan pdpAggregateTask := pdp.NewAggregatePDPDealTask(db, sc) pdpCache := pdp.NewTaskPDPSaveCache(db, dependencies.CachedPieceReader, iStore) + commPTask := pdp.NewPDPCommpTask(db, sc, cfg.Subsystems.CommPMaxTasks) - activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, pdpAddRoot, addProofSetTask, pdpAggregateTask, pdpCache, pdpDelRoot, pdpDelProofSetTask) + activeTasks = append(activeTasks, pdpNotifTask, pdpProveTask, pdpNextProvingPeriodTask, pdpInitProvingPeriodTask, commPTask, pdpAddRoot, addProofSetTask, pdpAggregateTask, pdpCache, pdpDelRoot, pdpDelProofSetTask) } idxMax := taskhelp.Max(cfg.Subsystems.IndexingMaxTasks) @@ -331,6 +332,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan activeTasks = append(activeTasks, ipniTask, indexingTask, pdpIdxTask, pdpIPNITask) if cfg.HTTP.Enable { + if !cfg.Subsystems.EnableDealMarket { + return nil, xerrors.New("deal market must be enabled on HTTP server") + } err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps) if err != nil { return nil, xerrors.Errorf("failed to start the HTTP server: %w", err) diff --git a/cmd/curio/toolbox.go b/cmd/curio/toolbox.go index a52d57073..c9333a98b 100644 --- a/cmd/curio/toolbox.go +++ b/cmd/curio/toolbox.go @@ -233,7 +233,9 @@ func filfoxMessage(cid string) (FilfoxMsg, error) { return FilfoxMsg{}, xerrors.Errorf("request failed: %w", err) } - defer res.Body.Close() + defer func() { + _ = res.Body.Close() + }() if res.StatusCode != 200 { return FilfoxMsg{}, xerrors.Errorf("request failed with status code %d", res.StatusCode) } diff --git a/cmd/curio/unseal.go b/cmd/curio/unseal.go index bfdcd8532..d09606334 100644 --- a/cmd/curio/unseal.go +++ b/cmd/curio/unseal.go @@ -328,7 +328,9 @@ var listUnsealPipelineCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to create output file: %w", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() writer = csv.NewWriter(file) } defer writer.Flush() diff --git a/cmd/pdptool/main.go b/cmd/pdptool/main.go index 8e634abde..c0699a3fa 100644 --- a/cmd/pdptool/main.go +++ b/cmd/pdptool/main.go @@ -121,7 +121,9 @@ var authCreateServiceSecretCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open pdpservice.json for writing: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() encoder := json.NewEncoder(file) if err := encoder.Encode(&serviceSecret); err != nil { return fmt.Errorf("failed to write to pdpservice.json: %v", err) @@ -210,7 +212,9 @@ var pingCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Check the response if resp.StatusCode == http.StatusOK { @@ -261,7 +265,9 @@ func loadPrivateKey() (*ecdsa.PrivateKey, error) { if err != nil { return nil, fmt.Errorf("failed to open pdpservice.json: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() var serviceSecret map[string]string decoder := json.NewDecoder(file) if err := decoder.Decode(&serviceSecret); err != nil { @@ -341,7 +347,9 @@ var piecePrepareCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open input file: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() // Get the piece size from flag or use file size fi, err := file.Stat() @@ -402,8 +410,8 @@ func startLocalNotifyServer() (string, chan struct{}, error) { }() defer func() { - server.Close() - ln.Close() + _ = server.Close() + _ = ln.Close() }() return serverAddr, notifyReceived, nil } @@ -422,9 +430,12 @@ func uploadOnePiece(client *http.Client, serviceURL string, reqBody []byte, jwtT if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() - if resp.StatusCode == http.StatusOK { + switch resp.StatusCode { + case http.StatusOK: if verbose { fmt.Println("http.StatusOK") } @@ -439,7 +450,7 @@ func uploadOnePiece(client *http.Client, serviceURL string, reqBody []byte, jwtT fmt.Printf("Piece already exists on the server. Piece CID: %s\n", pieceCID) } return nil - } else if resp.StatusCode == http.StatusCreated { + case http.StatusCreated: if verbose { fmt.Println("http.StatusCreated") } @@ -466,7 +477,9 @@ func uploadOnePiece(client *http.Client, serviceURL string, reqBody []byte, jwtT if err != nil { return fmt.Errorf("failed to upload piece data: %v", err) } - defer uploadResp.Body.Close() + defer func() { + _ = uploadResp.Body.Close() + }() if uploadResp.StatusCode != http.StatusNoContent { body, _ := io.ReadAll(uploadResp.Body) @@ -480,7 +493,7 @@ func uploadOnePiece(client *http.Client, serviceURL string, reqBody []byte, jwtT } return nil - } else { + default: body, _ := io.ReadAll(resp.Body) return fmt.Errorf("server returned status code %d: %s", resp.StatusCode, string(body)) } @@ -567,7 +580,9 @@ var pieceUploadCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open input file: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() // Get the piece size fi, err := file.Stat() @@ -696,7 +711,9 @@ var uploadFileCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to create chunk file: %v", err) } - defer chunkFile.Close() + defer func() { + _ = chunkFile.Close() + }() } if jwtToken == "" { if serviceName == "" { @@ -714,7 +731,9 @@ var uploadFileCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open input file: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() // Get the file size fi, err := file.Stat() @@ -809,7 +828,7 @@ var uploadFileCmd = &cli.Command{ } } if chunkFile != nil { - if _, err := chunkFile.Write([]byte(fmt.Sprintf("%s\n", commP))); err != nil { + if _, err := fmt.Fprintf(chunkFile, "%s\n", commP); err != nil { return fmt.Errorf("failed to write chunk to file: %v", err) } } @@ -922,7 +941,9 @@ var createProofSetCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Read and display the response bodyBytes, err := io.ReadAll(resp.Body) @@ -1000,7 +1021,9 @@ var getProofSetStatusCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Read and process the response bodyBytes, err := io.ReadAll(resp.Body) @@ -1101,7 +1124,9 @@ var getProofSetCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Read and process the response bodyBytes, err := io.ReadAll(resp.Body) @@ -1269,7 +1294,9 @@ var addRootsCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to send request: %v", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Read and display the response bodyBytes, err := io.ReadAll(resp.Body) @@ -1319,14 +1346,18 @@ var downloadFileCmd = &cli.Command{ if err != nil { return fmt.Errorf("failed to open chunk file: %v", err) } - defer chunkFile.Close() + defer func() { + _ = chunkFile.Close() + }() // Open the output file for writing outputFile, err := os.Create(outputFileName) if err != nil { return fmt.Errorf("failed to create output file: %v", err) } - defer outputFile.Close() + defer func() { + _ = outputFile.Close() + }() // Read all CIDs from the chunk file var cids []string @@ -1369,13 +1400,13 @@ var downloadFileCmd = &cli.Command{ // Check response status if resp.StatusCode != http.StatusOK { - resp.Body.Close() + _ = resp.Body.Close() return fmt.Errorf("failed to download piece %s: status code %d", cidString, resp.StatusCode) } // Stream the response body to the output file _, err = io.Copy(outputFile, resp.Body) - resp.Body.Close() + _ = resp.Body.Close() if err != nil { return fmt.Errorf("failed to write piece %s to output file: %v", cidString, err) } diff --git a/cmd/sptool/evm.go b/cmd/sptool/evm.go index 7b72b8199..64c7cb3a4 100644 --- a/cmd/sptool/evm.go +++ b/cmd/sptool/evm.go @@ -97,7 +97,7 @@ func getAddressAllowanceOnContract(ctx context.Context, api api.Gateway, wallet // Parse the contract ABI parsedABI, err := eabi.JSON(strings.NewReader(contractABI)) if err != nil { - return nil, fmt.Errorf("Failed to parse contract ABI: %w", err) + return nil, fmt.Errorf("failed to parse contract ABI: %w", err) } // Convert from Filecoin to Eth Address @@ -137,7 +137,7 @@ func getAddressAllowanceOnContract(ctx context.Context, api api.Gateway, wallet } if result.MsgRct.ExitCode.IsError() { - return nil, fmt.Errorf("Checking allowance failed with ExitCode %d", result.MsgRct.ExitCode) + return nil, fmt.Errorf("checking allowance failed with ExitCode %d", result.MsgRct.ExitCode) } // Decode return value (cbor -> evm ABI -> math/big Int -> filecoin big Int) @@ -158,7 +158,7 @@ func buildTransferViaEVMParams(amount *big.Int, receiverParams []byte) ([]byte, // Parse the contract's ABI parsedABI, err := eabi.JSON(strings.NewReader(contractABI)) if err != nil { - return nil, fmt.Errorf("Failed to parse contract ABI: %w", err) + return nil, fmt.Errorf("failed to parse contract ABI: %w", err) } // convert amount from Filecoin big.Int to math/big Int diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index ee2ec48f5..a0994aeee 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -539,7 +539,9 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error { if err != nil { return xerrors.Errorf("failed to open stream to peer %s: %w", addrInfo.ID, err) } - defer s.Close() + defer func() { + _ = s.Close() + }() if err := doRpc(ctx, s, &dealParams, &resp); err != nil { return xerrors.Errorf("send proposal rpc: %w", err) @@ -673,7 +675,9 @@ func doHttp(urls []*url.URL, deal interface{}, response interface{}) error { log.Warnw("failed to send request", "url", s, "error", err) continue } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { log.Warnw("failed to send request", "url", s, "status", resp.StatusCode) continue @@ -701,7 +705,7 @@ var initCmd = &cli.Command{ return err } - os.Mkdir(sdir, 0755) //nolint:errcheck + _ = os.Mkdir(sdir, 0755) //nolint:errcheck n, err := Setup(cctx.String(mk12_client_repo.Name)) if err != nil { @@ -786,7 +790,7 @@ var walletNew = &cli.Command{ out := map[string]interface{}{ "address": nk.String(), } - PrintJson(out) //nolint:errcheck + _ = PrintJson(out) //nolint:errcheck } else { fmt.Println(nk.String()) } @@ -927,7 +931,7 @@ var walletList = &cli.Command{ if !cctx.Bool("json") && dcap == nil { wallet[dataCapKey] = "X" } else if dcap != nil { - wallet[dataCapKey] = humanize.IBytes(dcap.Int.Uint64()) + wallet[dataCapKey] = humanize.IBytes(dcap.Uint64()) } } else { wallet[dataCapKey] = "n/a" @@ -1456,7 +1460,9 @@ var dealStatusCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to make HTTP request: %w", err) } - defer hresp.Body.Close() + defer func() { + _ = hresp.Body.Close() + }() if hresp.StatusCode != http.StatusOK { return xerrors.Errorf("HTTP request failed with status %d", hresp.StatusCode) } @@ -1829,7 +1835,9 @@ var mk20DealCmd = &cli.Command{ if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() @@ -2135,7 +2143,7 @@ var mk20ClientChunkUploadCmd = &cli.Command{ // Calculate the number of chunks numChunks := int((size + chunkSize - 1) / chunkSize) - f.Close() + _ = f.Close() api, closer, err := lcli.GetGatewayAPIV1(cctx) if err != nil { @@ -2221,7 +2229,9 @@ var mk20ClientChunkUploadCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to open file: %w", err) } - defer x.Close() + defer func() { + _ = x.Close() + }() for { gc, err := http.NewRequest("GET", purl.String()+"/market/mk20/uploads/"+dealid.String(), nil) @@ -2480,7 +2490,7 @@ var mk20PDPDealCmd = &cli.Command{ rootIDs := cctx.Uint64Slice("root-id") proofSetSet := cctx.IsSet("proofset-id") proofsetID := cctx.Uint64("proofset-id") - if !(addRoot || removeRoot || addProofset || removeProofset) { + if !addRoot && !removeRoot && !addProofset && !removeProofset { return xerrors.Errorf("at least one of --add-root, --remove-root, --add-proofset, --remove-proofset must be set") } @@ -2533,7 +2543,9 @@ var mk20PDPDealCmd = &cli.Command{ if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() @@ -2804,7 +2816,9 @@ var mk20ClientUploadCmd = &cli.Command{ return xerrors.Errorf("opening file: %w", err) } - defer f.Close() + defer func() { + _ = f.Close() + }() stat, err := f.Stat() if err != nil { diff --git a/cmd/sptool/toolbox_deal_tools.go b/cmd/sptool/toolbox_deal_tools.go index 4c9274a00..6fb83e41b 100644 --- a/cmd/sptool/toolbox_deal_tools.go +++ b/cmd/sptool/toolbox_deal_tools.go @@ -520,7 +520,9 @@ var allocateCmd = &cli.Command{ if err != nil { return err } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() diff --git a/cuhttp/server.go b/cuhttp/server.go index 1d3f00e5a..c315f6f75 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -2,6 +2,7 @@ package cuhttp import ( "context" + "errors" "fmt" "net/http" "strings" @@ -166,19 +167,19 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { // Use http.ServeMux as a fallback for routes not handled by chi chiRouter.NotFound(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - fmt.Fprintf(w, "Requested resource not found") + _, _ = fmt.Fprintf(w, "Requested resource not found") }) // Root path handler (simpler routes handled by http.ServeMux) chiRouter.Get("/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "Hello, World!\n -Curio\n") + _, _ = fmt.Fprintf(w, "Hello, World!\n -Curio\n") }) // Status endpoint to check the health of the service chiRouter.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "Service is up and running") + _, _ = fmt.Fprintf(w, "Service is up and running") }) // TODO: Attach a info page here with details about all the service and endpoints @@ -246,7 +247,7 @@ func (c cache) Get(ctx context.Context, key string) ([]byte, error) { var ret []byte err := c.db.QueryRow(ctx, `SELECT v FROM autocert_cache WHERE k = $1`, key).Scan(&ret) if err != nil { - if err == pgx.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { return nil, autocert.ErrCacheMiss } diff --git a/deps/apiinfo.go b/deps/apiinfo.go index 7fc2943b1..50715ce4c 100644 --- a/deps/apiinfo.go +++ b/deps/apiinfo.go @@ -75,7 +75,7 @@ func GetFullNodeAPIV1Curio(ctx *cli.Context, ainfoCfg []string) (api.Chain, json } // Compare with binary's network using BuildTypeString() - if !(strings.HasPrefix(string(networkName), "test") || strings.HasPrefix(string(networkName), "local")) { + if !strings.HasPrefix(string(networkName), "test") && !strings.HasPrefix(string(networkName), "local") { if networkName == "calibrationnet" { networkName = "calibnet" } diff --git a/deps/config/load.go b/deps/config/load.go index c2e4cd4c6..3ccf7601e 100644 --- a/deps/config/load.go +++ b/deps/config/load.go @@ -96,7 +96,7 @@ func FromReader(reader io.Reader, def interface{}, opts ...LoadCfgOpt) (interfac } for _, d := range movedFields { if md.IsDefined(d.Field...) { - fmt.Fprintf( + _, _ = fmt.Fprintf( warningOut, "WARNING: Use of deprecated configuration option '%s' will be removed in a future release, use '%s' instead\n", strings.Join(d.Field, "."), diff --git a/deps/config/old_lotus_miner.go b/deps/config/old_lotus_miner.go index b7f7febc7..53bb84879 100644 --- a/deps/config/old_lotus_miner.go +++ b/deps/config/old_lotus_miner.go @@ -692,8 +692,8 @@ func DefaultStorageMiner() *StorageMiner { }, } - cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" - cfg.Common.API.RemoteListenAddress = "127.0.0.1:2345" + cfg.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" + cfg.API.RemoteListenAddress = "127.0.0.1:2345" return cfg } diff --git a/harmony/harmonydb/sql/20250505-market-mk20.sql b/harmony/harmonydb/sql/20250505-market-mk20.sql index a1398e1d5..4a0a6c921 100644 --- a/harmony/harmonydb/sql/20250505-market-mk20.sql +++ b/harmony/harmonydb/sql/20250505-market-mk20.sql @@ -260,10 +260,9 @@ CREATE TABLE market_mk20_upload_waiting ( CREATE TABLE market_mk20_download_pipeline ( id TEXT NOT NULL, product TEXT NOT NULL, -- This allows us to run multiple refs per product for easier lifecycle management - piece_cid TEXT NOT NULL, -- This is pieceCid V1 to allow easy table lookups - piece_size BIGINT NOT NULL, + piece_cid_v2 TEXT NOT NULL, ref_ids BIGINT[] NOT NULL, - PRIMARY KEY (id, product, piece_cid, piece_size) + PRIMARY KEY (id, product, piece_cid_v2) ); -- Offline URLs for PoRep deals. @@ -363,7 +362,7 @@ CREATE TRIGGER trg_ready_at_chunks_update EXECUTE FUNCTION set_ready_at_when_all_chunks_complete(); -- This function triggers a download for an offline piece. --- It is different from MK1.2 PoRep pipeline as it download the offline pieces +-- It is different from MK1.2 PoRep pipeline as it downloads the offline pieces -- locally. This is to allow serving retrievals with piece park. CREATE OR REPLACE FUNCTION process_offline_download( _id TEXT, @@ -396,13 +395,13 @@ BEGIN FROM market_mk20_pipeline WHERE id = _id AND piece_cid_v2 = _piece_cid_v2 LIMIT 1; - -- 3. Look for existing piece + -- 3. Look for an existing piece SELECT id INTO _piece_id FROM parked_pieces WHERE piece_cid = _piece_cid AND piece_padded_size = _piece_size; - -- 4. Insert piece if not found + -- 4. Insert piece if it is not found IF NOT FOUND THEN INSERT INTO parked_pieces (piece_cid, piece_padded_size, piece_raw_size, long_term) VALUES (_piece_cid, _piece_size, _raw_size, NOT (_deal_aggregation > 0)) @@ -415,9 +414,9 @@ BEGIN RETURNING ref_id INTO _ref_id; -- 6. Insert or update download pipeline with ref_id - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) - VALUES (_id, _piece_cid, _piece_size, _product, ARRAY[_ref_id]) - ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE + INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) + VALUES (_id, _piece_cid_v2, _product, ARRAY[_ref_id]) + ON CONFLICT (id, piece_cid_v2, product) DO UPDATE SET ref_ids = ( SELECT ARRAY( SELECT DISTINCT r @@ -550,6 +549,9 @@ CREATE TABLE pdp_pipeline ( downloaded BOOLEAN DEFAULT FALSE, + commp_task_id BIGINT DEFAULT NULL, + after_commp BOOLEAN DEFAULT FALSE, + deal_aggregation INT NOT NULL DEFAULT 0, aggr_index BIGINT DEFAULT 0, agg_task_id BIGINT DEFAULT NULL, @@ -582,6 +584,69 @@ CREATE TABLE pdp_pipeline ( PRIMARY KEY (id, aggr_index) ); +-- This function is used to mark a piece as downloaded in pdp_pipeline +-- A deal with multiple HTTP sources will have multiple ref_ids, +-- and download is handled by market_mk20_download_pipeline table +-- We add ref_id to pdp_pipeline once download is successful. +create or replace function mk20_pdp_mark_downloaded(_product text) +returns integer +language plpgsql +as $$ +declare + updated_count int := 0; +begin + with candidates as ( + select p.id, p.piece_cid_v2, dp.ref_ids + from pdp_pipeline p + join market_mk20_download_pipeline dp + on dp.id = p.id + and dp.piece_cid_v2 = p.piece_cid_v2 + and dp.product = _product + where p.piece_ref is null + ), + picked as ( + -- choose ONE completed ref_id from the array for each (id,piece_cid_v2) + select c.id, c.piece_cid_v2, c.ref_ids, ch.ref_id as chosen_ref + from candidates c + cross join lateral ( + select pr.ref_id + from unnest(c.ref_ids) as r(ref_id) + join parked_piece_refs pr on pr.ref_id = r.ref_id + join parked_pieces pp on pp.id = pr.piece_id + where pp.complete = true + limit 1 + ) ch + ), + del_other_refs as ( + delete from parked_piece_refs pr + using picked + where pr.ref_id = any(picked.ref_ids) + and pr.ref_id != picked.chosen_ref + returning 1 + ), + del_download_rows as ( + delete from market_mk20_download_pipeline dp + using picked + where dp.id = picked.id + and dp.piece_cid_v2 = picked.piece_cid_v2 + and dp.product = _product + returning 1 + ), + upd as ( + update pdp_pipeline p + set downloaded = true, + piece_ref = picked.chosen_ref + from picked + where p.id = picked.id + and p.piece_cid_v2 = picked.piece_cid_v2 + returning 1 + ) + select count(*) into updated_count from upd; + + return updated_count; +end; +$$; + CREATE TABLE market_mk20_clients ( client TEXT PRIMARY KEY, allowed BOOLEAN DEFAULT TRUE @@ -752,3 +817,63 @@ END; $$ LANGUAGE plpgsql; +create or replace function mk20_ddo_mark_downloaded(_product text) +returns integer +language plpgsql +as $$ +declare +updated_count int := 0; +begin + with candidates as ( + select p.id, p.piece_cid_v2, dp.ref_ids + from market_mk20_pipeline p + join market_mk20_download_pipeline dp + on dp.id = p.id + and dp.piece_cid_v2 = p.piece_cid_v2 + and dp.product = _product + where p.piece_ref is null + ), + picked as ( + -- choose ONE completed ref_id from the array for each (id,piece_cid_v2) + select c.id, c.piece_cid_v2, c.ref_ids, ch.ref_id as chosen_ref + from candidates c + cross join lateral ( + select pr.ref_id + from unnest(c.ref_ids) as r(ref_id) + join parked_piece_refs pr on pr.ref_id = r.ref_id + join parked_pieces pp on pp.id = pr.piece_id + where pp.complete = true + limit 1 + ) ch + ), + del_other_refs as ( + delete from parked_piece_refs pr + using picked + where pr.ref_id = any(picked.ref_ids) + and pr.ref_id != picked.chosen_ref + returning 1 + ), + del_download_rows as ( + delete from market_mk20_download_pipeline dp + using picked + where dp.id = picked.id + and dp.piece_cid_v2 = picked.piece_cid_v2 + and dp.product = _product + returning 1 + ), + upd as ( + update market_mk20_pipeline p + set downloaded = true, + url = 'pieceref:' || picked.chosen_ref::text + from picked + where p.id = picked.id + and p.piece_cid_v2 = picked.piece_cid_v2 + returning 1 + ) + select count(*) into updated_count from upd; + + return updated_count; +end; +$$; + + diff --git a/harmony/harmonydb/userfuncs.go b/harmony/harmonydb/userfuncs.go index 80663ef2d..00d61db36 100644 --- a/harmony/harmonydb/userfuncs.go +++ b/harmony/harmonydb/userfuncs.go @@ -121,7 +121,7 @@ func (d dbscanRows) Close() error { return nil } func (d dbscanRows) Columns() ([]string, error) { - return lo.Map(d.Rows.FieldDescriptions(), func(fd pgconn.FieldDescription, _ int) string { + return lo.Map(d.FieldDescriptions(), func(fd pgconn.FieldDescription, _ int) string { return fd.Name }), nil } diff --git a/harmony/harmonytask/harmonytask.go b/harmony/harmonytask/harmonytask.go index 48f045f80..c03aa3e5b 100644 --- a/harmony/harmonytask/harmonytask.go +++ b/harmony/harmonytask/harmonytask.go @@ -173,7 +173,7 @@ func New( grace: grace, db: db, reg: reg, - ownerID: reg.Resources.MachineID, // The current number representing "hostAndPort" + ownerID: reg.MachineID, // The current number representing "hostAndPort" taskMap: make(map[string]*taskTypeHandler, len(impls)), follows: make(map[string][]followStruct), hostAndPort: hostnameAndPort, @@ -190,8 +190,8 @@ func New( } h.Max = h.Max.Instance() - if Registry[h.TaskTypeDetails.Name] == nil { - return nil, fmt.Errorf("task %s not registered: var _ = harmonytask.Reg(t TaskInterface)", h.TaskTypeDetails.Name) + if Registry[h.Name] == nil { + return nil, fmt.Errorf("task %s not registered: var _ = harmonytask.Reg(t TaskInterface)", h.Name) } if len(h.Name) > 16 { @@ -199,7 +199,7 @@ func New( } e.handlers = append(e.handlers, &h) - e.taskMap[h.TaskTypeDetails.Name] = &h + e.taskMap[h.Name] = &h } // resurrect old work @@ -248,31 +248,31 @@ func (e *TaskEngine) GracefullyTerminate() { for { timeout := time.Millisecond for _, h := range e.handlers { - if h.TaskTypeDetails.Name == "WinPost" && h.Max.Active() > 0 { + if h.Name == "WinPost" && h.Max.Active() > 0 { timeout = time.Second log.Infof("node shutdown deferred for %f seconds", timeout.Seconds()) continue } - if h.TaskTypeDetails.Name == "WdPost" && h.Max.Active() > 0 { + if h.Name == "WdPost" && h.Max.Active() > 0 { timeout = time.Second * 3 log.Infof("node shutdown deferred for %f seconds due to running WdPost task", timeout.Seconds()) continue } - if h.TaskTypeDetails.Name == "WdPostSubmit" && h.Max.Active() > 0 { + if h.Name == "WdPostSubmit" && h.Max.Active() > 0 { timeout = time.Second log.Infof("node shutdown deferred for %f seconds due to running WdPostSubmit task", timeout.Seconds()) continue } - if h.TaskTypeDetails.Name == "WdPostRecover" && h.Max.Active() > 0 { + if h.Name == "WdPostRecover" && h.Max.Active() > 0 { timeout = time.Second log.Infof("node shutdown deferred for %f seconds due to running WdPostRecover task", timeout.Seconds()) continue } // Test tasks for itest - if h.TaskTypeDetails.Name == "ThingOne" && h.Max.Active() > 0 { + if h.Name == "ThingOne" && h.Max.Active() > 0 { timeout = time.Second log.Infof("node shutdown deferred for %f seconds due to running itest task", timeout.Seconds()) continue @@ -388,13 +388,13 @@ func (e *TaskEngine) pollerTryAllWork(schedulable bool) bool { } for _, v := range e.handlers { if !schedulable { - if v.TaskTypeDetails.SchedulingOverrides == nil { + if v.SchedulingOverrides == nil { continue } // Override the schedulable flag if the task has any assigned overrides var foundOverride bool - for relatedTaskName := range v.TaskTypeDetails.SchedulingOverrides { + for relatedTaskName := range v.SchedulingOverrides { var assignedOverrideTasks []int err := e.db.Select(e.ctx, &assignedOverrideTasks, `SELECT id FROM harmony_task @@ -466,9 +466,9 @@ func (e *TaskEngine) pollerTryAllWork(schedulable bool) bool { if v.AssertMachineHasCapacity() != nil { continue } - if v.TaskTypeDetails.IAmBored != nil { + if v.IAmBored != nil { var added []TaskID - err := v.TaskTypeDetails.IAmBored(func(extraInfo func(TaskID, *harmonydb.Tx) (shouldCommit bool, seriousError error)) { + err := v.IAmBored(func(extraInfo func(TaskID, *harmonydb.Tx) (shouldCommit bool, seriousError error)) { v.AddTask(func(tID TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { b, err := extraInfo(tID, tx) if err == nil && shouldCommit { diff --git a/harmony/harmonytask/task_type_handler.go b/harmony/harmonytask/task_type_handler.go index f3af543c3..2a0aafa36 100644 --- a/harmony/harmonytask/task_type_handler.go +++ b/harmony/harmonytask/task_type_handler.go @@ -119,8 +119,8 @@ canAcceptAgain: releaseStorage := func() { } - if h.TaskTypeDetails.Cost.Storage != nil { - markComplete, err := h.TaskTypeDetails.Cost.Storage.Claim(int(*tID)) + if h.Cost.Storage != nil { + markComplete, err := h.Cost.Claim(int(*tID)) if err != nil { log.Infow("did not accept task", "task_id", strconv.Itoa(int(*tID)), "reason", "storage claim failed", "name", h.Name, "error", err) @@ -354,8 +354,8 @@ func (h *taskTypeHandler) AssertMachineHasCapacity() error { return xerrors.Errorf("Did not accept %s task: out of available GPU: required %f available %f)", h.Name, h.Cost.Gpu, r.Gpu) } - if h.TaskTypeDetails.Cost.Storage != nil { - if !h.TaskTypeDetails.Cost.Storage.HasCapacity() { + if h.Cost.Storage != nil { + if !h.Cost.HasCapacity() { return errors.New("Did not accept " + h.Name + " task: out of available Storage") } } diff --git a/itests/pdp_prove_test.go b/itests/pdp_prove_test.go index 98f0bf6a8..88aea6ae7 100644 --- a/itests/pdp_prove_test.go +++ b/itests/pdp_prove_test.go @@ -12,7 +12,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/lib/proof" - _ "github.com/filecoin-project/curio/lib/proof" "github.com/filecoin-project/curio/lib/testutils" "github.com/filecoin-project/curio/pdp/contract" "github.com/filecoin-project/curio/tasks/pdp" @@ -32,7 +31,9 @@ func TestPDPProving(t *testing.T) { fileStr, err := testutils.CreateRandomTmpFile(dir, rawSize) require.NoError(t, err) - defer os.Remove(fileStr) + defer func() { + _ = os.Remove(fileStr) + }() f, err := os.Open(fileStr) require.NoError(t, err) diff --git a/lib/cachedreader/cachedreader.go b/lib/cachedreader/cachedreader.go index a476ae417..179eb6862 100644 --- a/lib/cachedreader/cachedreader.go +++ b/lib/cachedreader/cachedreader.go @@ -23,7 +23,7 @@ import ( "github.com/filecoin-project/curio/market/indexstore" ) -var NoDealErr = errors.New("no deals found") +var ErrNoDeal = errors.New("no deals found") var log = logging.Logger("cached-reader") @@ -172,7 +172,7 @@ func (cpr *CachedPieceReader) getPieceReaderFromSector(ctx context.Context, piec } if len(deals) == 0 { - return nil, 0, fmt.Errorf("piece cid %s: %w", pieceCid, NoDealErr) + return nil, 0, fmt.Errorf("piece cid %s: %w", pieceCid, ErrNoDeal) } // For each deal, try to read an unsealed copy of the data from the sector diff --git a/lib/cachedreader/prefetch_test.go b/lib/cachedreader/prefetch_test.go index d92b1f371..cafe3489c 100644 --- a/lib/cachedreader/prefetch_test.go +++ b/lib/cachedreader/prefetch_test.go @@ -49,7 +49,9 @@ func TestPrefetchReader_BasicRead(t *testing.T) { testData := []byte("Hello, World!") source := &mockReader{data: testData} reader := New(source, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() buf := make([]byte, len(testData)) n, err := reader.Read(buf) @@ -67,7 +69,9 @@ func TestPrefetchReader_BasicRead(t *testing.T) { func TestPrefetchReader_ReadEmpty(t *testing.T) { reader := New(&mockReader{data: []byte{}}, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() buf := make([]byte, 10) n, err := reader.Read(buf) @@ -82,7 +86,9 @@ func TestPrefetchReader_ReadEmpty(t *testing.T) { func TestPrefetchReader_ReadZeroLength(t *testing.T) { reader := New(&mockReader{data: []byte("data")}, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() n, err := reader.Read([]byte{}) @@ -103,7 +109,9 @@ func TestPrefetchReader_LargeRead(t *testing.T) { source := &mockReader{data: testData} reader := New(source, 4096) // Smaller buffer to test multiple reads - defer reader.Close() + defer func() { + _ = reader.Close() + }() // Read in chunks buf := make([]byte, len(testData)) @@ -145,7 +153,9 @@ func TestPrefetchReader_SlowSource(t *testing.T) { } reader := New(source, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() buf := make([]byte, len(testData)) n, err := reader.Read(buf) @@ -169,7 +179,9 @@ func TestPrefetchReader_ErrorHandling(t *testing.T) { } reader := New(source, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() buf := make([]byte, len(testData)) _, firstErr := reader.Read(buf) @@ -209,7 +221,9 @@ func TestPrefetchReader_Close(t *testing.T) { func TestPrefetchReader_MinBufferSize(t *testing.T) { reader := New(&mockReader{}, 100) // Less than minBufferDepth - defer reader.Close() + defer func() { + _ = reader.Close() + }() if cap(reader.buffer) < minBufferDepth { t.Errorf("buffer size %d is less than minimum %d", cap(reader.buffer), minBufferDepth) @@ -223,7 +237,9 @@ func TestPrefetchReader_ReadAfterError(t *testing.T) { } reader := New(source, 1024) - defer reader.Close() + defer func() { + _ = reader.Close() + }() buf := make([]byte, 4) _, firstErr := reader.Read(buf) diff --git a/lib/chainsched/chain_sched_test.go b/lib/chainsched/chain_sched_test.go index 20a04d0d4..5f378c9df 100644 --- a/lib/chainsched/chain_sched_test.go +++ b/lib/chainsched/chain_sched_test.go @@ -408,9 +408,10 @@ func TestTimeoutResubscription(t *testing.T) { count := callCount mu.Unlock() - if count == 1 { + switch count { + case 1: defer close(firstCallCh) - } else if count == 2 { + case 2: defer close(secondCallCh) } @@ -484,9 +485,10 @@ func TestMultipleChanges(t *testing.T) { callCount++ lastApply = apply - if callCount == 1 { + switch callCount { + case 1: close(firstCallDone) - } else if callCount == 2 { + case 2: close(secondCallDone) } return nil diff --git a/lib/fastparamfetch/paramfetch.go b/lib/fastparamfetch/paramfetch.go index 16344a886..0f98d8e30 100644 --- a/lib/fastparamfetch/paramfetch.go +++ b/lib/fastparamfetch/paramfetch.go @@ -212,7 +212,9 @@ func (ft *fetch) checkFile(path string, info paramFile) error { if err != nil { return err } - defer f.Close() + defer func() { + _ = f.Close() + }() h := blake2b.New512() if _, err := io.Copy(h, f); err != nil { @@ -276,7 +278,9 @@ func doFetch(ctx context.Context, out string, info paramFile) error { if err != nil { return err } - defer outf.Close() + defer func() { + _ = outf.Close() + }() fStat, err := outf.Stat() if err != nil { @@ -296,7 +300,9 @@ func doFetch(ctx context.Context, out string, info paramFile) error { if err != nil { return err } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() _, err = io.Copy(outf, resp.Body) diff --git a/lib/ffi/piece_funcs.go b/lib/ffi/piece_funcs.go index 882f99aa8..1582d756e 100644 --- a/lib/ffi/piece_funcs.go +++ b/lib/ffi/piece_funcs.go @@ -80,11 +80,15 @@ func (sb *SealCalls) RemovePiece(ctx context.Context, id storiface.PieceNumber) func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType, verifySize bool) (abi.PieceInfo, uint64, error) { // Use storageType in AcquireSector - paths, _, done, err := sb.Sectors.AcquireSector(ctx, nil, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType) + paths, pathIDs, done, err := sb.Sectors.AcquireSector(ctx, nil, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType) if err != nil { return abi.PieceInfo{}, 0, err } - defer done() + skipDeclare := storiface.FTPiece + + defer func() { + done(skipDeclare) + }() dest := paths.Piece tempDest := dest + storiface.TempSuffix @@ -142,6 +146,13 @@ func (sb *SealCalls) WriteUploadPiece(ctx context.Context, pieceID storiface.Pie return abi.PieceInfo{}, 0, xerrors.Errorf("rename temp piece to dest %s -> %s: %w", tempDest, dest, err) } + skipDeclare = storiface.FTNone + removeTemp = false + + if err := sb.ensureOneCopy(ctx, pieceID.Ref().ID, pathIDs, storiface.FTPiece); err != nil { + return abi.PieceInfo{}, 0, xerrors.Errorf("ensure one copy: %w", err) + } + return abi.PieceInfo{PieceCID: pcid, Size: psize}, uint64(n), nil } diff --git a/lib/ffi/scrub_funcs.go b/lib/ffi/scrub_funcs.go index 2091d641e..92e75fd57 100644 --- a/lib/ffi/scrub_funcs.go +++ b/lib/ffi/scrub_funcs.go @@ -22,7 +22,9 @@ func (sb *SealCalls) CheckUnsealedCID(ctx context.Context, s storiface.SectorRef if err != nil { return cid.Undef, xerrors.Errorf("getting unsealed sector reader: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() ssize, err := s.ProofType.SectorSize() if err != nil { diff --git a/lib/ffi/unseal_funcs.go b/lib/ffi/unseal_funcs.go index 778ce3664..5a703096d 100644 --- a/lib/ffi/unseal_funcs.go +++ b/lib/ffi/unseal_funcs.go @@ -40,7 +40,9 @@ func (sb *SealCalls) decodeCommon(ctx context.Context, taskID harmonytask.TaskID if err != nil { return xerrors.Errorf("creating unsealed file: %w", err) } - defer outFile.Close() + defer func() { + _ = outFile.Close() + }() start := time.Now() diff --git a/lib/hugepageutil/checkhuge.go b/lib/hugepageutil/checkhuge.go index 7ba2e503c..80855eda4 100644 --- a/lib/hugepageutil/checkhuge.go +++ b/lib/hugepageutil/checkhuge.go @@ -14,7 +14,9 @@ func CheckHugePages(minPages int) error { if err != nil { return xerrors.Errorf("error opening /proc/meminfo: %w", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) hugepagesTotal := 0 diff --git a/lib/panicreport/panic_reporter.go b/lib/panicreport/panic_reporter.go index 0fe99fe4d..4a83b6b79 100644 --- a/lib/panicreport/panic_reporter.go +++ b/lib/panicreport/panic_reporter.go @@ -147,10 +147,8 @@ func writeJournalTail(tailLen int, repoPath, file string) { } jScan := backscanner.New(j, int(js.Size())) linesWritten := 0 - for { - if linesWritten > tailLen { - break - } + for linesWritten <= tailLen { + line, _, err := jScan.LineBytes() if err != nil { if err != io.EOF { diff --git a/lib/paths/db_index.go b/lib/paths/db_index.go index b5cc10a46..5d778bcb5 100644 --- a/lib/paths/db_index.go +++ b/lib/paths/db_index.go @@ -522,7 +522,9 @@ func (dbi *DBIndex) batchStorageDeclareSectors(ctx context.Context, declarations } br := tx.SendBatch(ctx, batch) - defer br.Close() + defer func() { + _ = br.Close() + }() for i := 0; i < batch.Len(); i++ { _, err := br.Exec() diff --git a/lib/paths/http_handler.go b/lib/paths/http_handler.go index d04e46ce6..020afb8fd 100644 --- a/lib/paths/http_handler.go +++ b/lib/paths/http_handler.go @@ -321,7 +321,9 @@ func (handler *FetchHandler) remoteGetStash(w http.ResponseWriter, r *http.Reque } return } - defer readCloser.Close() + defer func() { + _ = readCloser.Close() + }() w.Header().Set("Content-Type", "application/octet-stream") _, err = io.Copy(w, readCloser) diff --git a/lib/paths/local.go b/lib/paths/local.go index dc83237fb..fe437f5ae 100644 --- a/lib/paths/local.go +++ b/lib/paths/local.go @@ -476,7 +476,7 @@ func (st *Local) declareSectors(ctx context.Context, p string, id storiface.ID, } for _, decl := range decls[id] { - for _, fileType := range decl.SectorFileType.AllSet() { + for _, fileType := range decl.AllSet() { indexed[storiface.Decl{ SectorID: decl.SectorID, SectorFileType: fileType, diff --git a/lib/proofsvc/clientctl.go b/lib/proofsvc/clientctl.go index cf5eb760c..00c343053 100644 --- a/lib/proofsvc/clientctl.go +++ b/lib/proofsvc/clientctl.go @@ -98,7 +98,9 @@ func CheckAvailability() (bool, error) { if err != nil { return false, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return false, xerrors.Errorf("failed to check availability: %s", resp.Status) @@ -148,7 +150,9 @@ func GetCurrentPrice() (PriceResponse, error) { if err != nil { return PriceResponse{}, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return PriceResponse{}, xerrors.Errorf("failed to get current price: %s", resp.Status) @@ -195,7 +199,9 @@ func UploadProofData(ctx context.Context, proofData []byte) (cid.Cid, error) { if err != nil { return cid.Undef, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) @@ -224,7 +230,9 @@ func RequestProof(request common.ProofRequest) (bool, error) { if err != nil { return false, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) @@ -265,7 +273,9 @@ func GetProofStatus(requestCid cid.Cid) (common.ProofResponse, error) { if err != nil { return common.ProofResponse{}, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) @@ -324,7 +334,9 @@ func GetClientPaymentStatus(walletID abi.ActorID) (*ClientPaymentStatus, error) if err != nil { return nil, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode == http.StatusNotFound { return nil, xerrors.Errorf("no payments found for wallet %d", walletID) diff --git a/lib/proofsvc/provictl.go b/lib/proofsvc/provictl.go index 2b7aaa343..3f471bcda 100644 --- a/lib/proofsvc/provictl.go +++ b/lib/proofsvc/provictl.go @@ -108,7 +108,9 @@ func CreateWorkAsk(ctx context.Context, resolver *AddressResolver, signer addres if err != nil { return 0, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) @@ -139,7 +141,9 @@ func PollWork(address string) (common.WorkResponse, error) { if err != nil { return common.WorkResponse{}, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return common.WorkResponse{}, xerrors.Errorf("failed to poll work: %s", resp.Status) @@ -175,7 +179,9 @@ func WithdrawAsk(ctx context.Context, resolver *AddressResolver, signer address. if err != nil { return xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { bodyBytes, _ := io.ReadAll(resp.Body) @@ -201,7 +207,9 @@ func GetProof(cid cid.Cid) ([]byte, error) { if err != nil { return nil, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return nil, xerrors.Errorf("failed to get proof: %s", resp.Status) @@ -241,7 +249,9 @@ func RespondWork(ctx context.Context, resolver *AddressResolver, address address if err != nil { return common.ProofReward{}, xerrors.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) diff --git a/lib/testutils/testutils.go b/lib/testutils/testutils.go index 6598dcad4..49b676bd4 100644 --- a/lib/testutils/testutils.go +++ b/lib/testutils/testutils.go @@ -115,7 +115,9 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, if err != nil { return cid.Undef, err } - defer file.Close() + defer func() { + _ = file.Close() + }() stat, err := file.Stat() if err != nil { @@ -243,18 +245,20 @@ func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggre if err != nil { return cid.Undef, err } - defer f.Close() + defer func() { + _ = f.Close() + }() cp := new(commp.Calc) w := io.MultiWriter(cp, f) n, err := io.Copy(w, out) if err != nil { - f.Close() + _ = f.Close() return cid.Undef, xerrors.Errorf("writing aggregate: %w", err) } - f.Close() + _ = f.Close() digest, paddedPieceSize, err := cp.Digest() if err != nil { @@ -279,9 +283,13 @@ func CreateAggregateFromCars(files []string, dealSize abi.PaddedPieceSize, aggre } if !aggregateOut { - defer os.Remove(f.Name()) + defer func() { + _ = os.Remove(f.Name()) + }() } else { - defer os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", comm.PCidV2().String())) //nolint:errcheck + defer func() { + _ = os.Rename(f.Name(), fmt.Sprintf("aggregate_%s.piece", comm.PCidV2().String())) + }() } return comm.PCidV2(), nil diff --git a/market/indexstore/indexstore.go b/market/indexstore/indexstore.go index 02e88ab53..36ec8154f 100644 --- a/market/indexstore/indexstore.go +++ b/market/indexstore/indexstore.go @@ -3,7 +3,6 @@ package indexstore import ( "context" "embed" - _ "embed" "errors" "fmt" "math/rand" diff --git a/market/ipni/chunker/initial-chunker.go b/market/ipni/chunker/initial-chunker.go index 40039b932..d96a8c3d7 100644 --- a/market/ipni/chunker/initial-chunker.go +++ b/market/ipni/chunker/initial-chunker.go @@ -235,7 +235,9 @@ func (c *InitialChunker) finishDB(ctx context.Context, db *harmonydb.DB, pieceCi // Send the batch br := tx.SendBatch(ctx, batch) - defer br.Close() + defer func() { + _ = br.Close() + }() // Execute the batch and check for errors for i := 0; i < totalChunks; i++ { @@ -297,7 +299,9 @@ func (c *InitialChunker) finishCAR(ctx context.Context, db *harmonydb.DB, pieceC // Send the batch br := tx.SendBatch(ctx, batch) - defer br.Close() + defer func() { + _ = br.Close() + }() // Execute the batch and check for errors for i := 0; i < totalChunks; i++ { diff --git a/market/libp2p/redirector.go b/market/libp2p/redirector.go index ad54563a0..f75938b5d 100644 --- a/market/libp2p/redirector.go +++ b/market/libp2p/redirector.go @@ -1,6 +1,7 @@ package libp2p import ( + "errors" "fmt" "net/http" "time" @@ -35,7 +36,7 @@ func (rp *Redirector) handleLibp2pWebsocket(w http.ResponseWriter, r *http.Reque var localListen string err := rp.db.QueryRow(r.Context(), "SELECT local_listen FROM libp2p").Scan(&localListen) if err != nil { - if err == pgx.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { http.Error(w, "Remote LibP2P host undefined", http.StatusBadGateway) return } @@ -61,7 +62,9 @@ func (rp *Redirector) handleLibp2pWebsocket(w http.ResponseWriter, r *http.Reque log.Infof("Error connecting to target WebSocket server: %v", err) return } - defer targetConn.Close() + defer func() { + _ = targetConn.Close() + }() // Upgrade the client connection to a WebSocket connection upgrader := websocket.Upgrader{ @@ -75,7 +78,9 @@ func (rp *Redirector) handleLibp2pWebsocket(w http.ResponseWriter, r *http.Reque log.Infof("WebSocket upgrade error: %v", err) return } - defer clientConn.Close() + defer func() { + _ = clientConn.Close() + }() // Proxy data between clientConn and targetConn errc := make(chan error, 2) diff --git a/market/mk12/mk12.go b/market/mk12/mk12.go index 80d3786ee..e3b01a8ef 100644 --- a/market/mk12/mk12.go +++ b/market/mk12/mk12.go @@ -40,7 +40,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - ctypes "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/storage/ctladdr" ) @@ -223,7 +222,7 @@ func (m *MK12) ExecuteDeal(ctx context.Context, dp *DealParams, clientPeer peer. valid := m.applyFilters(ctx, ds) if valid != nil && valid.error != nil { - log.Errorf("failed to apply filetrs: %s", valid.error.Error()) + log.Errorf("failed to apply filetrs: %s", valid.Error()) return &ProviderDealRejectionInfo{ Reason: "internal server error: failed to apply filters", }, nil @@ -320,7 +319,7 @@ func (m *MK12) validateDealProposal(ctx context.Context, deal *ProviderDealState return &validationError{error: err} } - bounds, err := m.api.StateDealProviderCollateralBounds(ctx, proposal.PieceSize, proposal.VerifiedDeal, ctypes.EmptyTSK) + bounds, err := m.api.StateDealProviderCollateralBounds(ctx, proposal.PieceSize, proposal.VerifiedDeal, types.EmptyTSK) if err != nil { return &validationError{ reason: "server error: getting collateral bounds", @@ -330,7 +329,7 @@ func (m *MK12) validateDealProposal(ctx context.Context, deal *ProviderDealState // The maximum amount of collateral that the provider will put into escrow // for a deal is calculated as a multiple of the minimum bounded amount - maxC := ctypes.BigMul(bounds.Min, ctypes.NewInt(maxDealCollateralMultiplier)) + maxC := types.BigMul(bounds.Min, types.NewInt(maxDealCollateralMultiplier)) pcMin := bounds.Min pcMax := maxC @@ -345,7 +344,7 @@ func (m *MK12) validateDealProposal(ctx context.Context, deal *ProviderDealState return &validationError{error: err} } - tsk, err := ctypes.TipSetKeyFromBytes(tok) + tsk, err := types.TipSetKeyFromBytes(tok) if err != nil { return &validationError{ reason: "server error: tip set key from bytes", @@ -490,7 +489,7 @@ func (m *MK12) processDeal(ctx context.Context, deal *ProviderDealState) (*Provi if !deal.IsOffline { // Reject incorrect sized online deals except verified deal less than 1 MiB because verified deals can be 1 MiB minimum even if rawSize is much lower - if deal.ClientDealProposal.Proposal.PieceSize != padreader.PaddedSize(deal.Transfer.Size).Padded() && !(deal.ClientDealProposal.Proposal.VerifiedDeal && deal.ClientDealProposal.Proposal.PieceSize <= abi.PaddedPieceSize(1<<20)) { + if deal.ClientDealProposal.Proposal.PieceSize != padreader.PaddedSize(deal.Transfer.Size).Padded() && (!deal.ClientDealProposal.Proposal.VerifiedDeal || deal.ClientDealProposal.Proposal.PieceSize > abi.PaddedPieceSize(1<<20)) { return &ProviderDealRejectionInfo{ Reason: fmt.Sprintf("deal proposal piece size %d doesn't match padded piece size %d", deal.ClientDealProposal.Proposal.PieceSize, padreader.PaddedSize(deal.Transfer.Size).Padded()), }, nil diff --git a/market/mk20/client/http_client.go b/market/mk20/client/http_client.go index 47bf81a05..72ad9cd2e 100644 --- a/market/mk20/client/http_client.go +++ b/market/mk20/client/http_client.go @@ -88,7 +88,9 @@ func (c *HTTPClient) do(ctx context.Context, method, p string, body io.Reader, v Error: err, } } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != 200 { msg, err := io.ReadAll(resp.Body) diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index 9a1f1c059..b1e2bd2db 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -24,7 +24,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" ) -var UnknowContract = errors.New("provider does not work with this market") +var ErrUnknowContract = errors.New("provider does not work with this market") // DDOV1 defines a structure for handling provider, client, and piece manager information with associated contract and notification details // for a DDO deal handling. @@ -34,29 +34,29 @@ type DDOV1 struct { Provider address.Address `json:"provider"` // Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs - PieceManager address.Address `json:"piece_manager"` + PieceManager address.Address `json:"pieceManager"` // Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. // It must be at least 518400 Duration abi.ChainEpoch `json:"duration"` // AllocationId represents an allocation identifier for the deal. - AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty" swaggertype:"integer" format:"uint64" example:"1"` + AllocationId *verifreg.AllocationId `json:"allocationId,omitempty" swaggertype:"integer" format:"uint64" example:"1"` // ContractAddress specifies the address of the contract governing the deal - ContractAddress string `json:"contract_address"` + ContractAddress string `json:"contractAddress"` // ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract - ContractVerifyMethod string `json:"contract_verify_method"` + ContractVerifyMethod string `json:"contractVerifyMethod"` // ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - ContractVerifyMethodParams []byte `json:"contract_verify_method_params,omitempty" swaggertype:"string" format:"byte"` + ContractVerifyMethodParams []byte `json:"contractVerifyMethodParams,omitempty" swaggertype:"string" format:"byte"` // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated - NotificationAddress string `json:"notification_address"` + NotificationAddress string `json:"notificationAddress"` // NotificationPayload holds the notification data typically in a serialized byte array format. - NotificationPayload []byte `json:"notification_payload,omitempty" swaggertype:"string" format:"byte"` + NotificationPayload []byte `json:"notificationPayload,omitempty" swaggertype:"string" format:"byte"` } func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { @@ -130,7 +130,7 @@ func (d *DDOV1) GetDealID(ctx context.Context, db *harmonydb.DB, eth *ethclient. err := db.QueryRow(ctx, `SELECT abi FROM ddo_contracts WHERE address = $1`, d.ContractAddress).Scan(&abiStr) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return -1, ErrMarketNotEnabled, UnknowContract + return -1, ErrMarketNotEnabled, ErrUnknowContract } return -1, ErrServerInternalError, xerrors.Errorf("getting abi: %w", err) } diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index cfb77ede3..b4e1bc11c 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -868,21 +868,21 @@ const docTemplate = `{ "mk20.DDOV1": { "type": "object", "properties": { - "allocation_id": { + "allocationId": { "description": "AllocationId represents an allocation identifier for the deal.", "type": "integer", "format": "uint64", "example": 1 }, - "contract_address": { + "contractAddress": { "description": "ContractAddress specifies the address of the contract governing the deal", "type": "string" }, - "contract_verify_method": { + "contractVerifyMethod": { "description": "ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract", "type": "string" }, - "contract_verify_method_params": { + "contractVerifyMethodParams": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", "type": "string", "format": "byte" @@ -891,16 +891,16 @@ const docTemplate = `{ "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", "type": "integer" }, - "notification_address": { + "notificationAddress": { "description": "NotificationAddress specifies the address to which notifications will be relayed to when sector is activated", "type": "string" }, - "notification_payload": { + "notificationPayload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", "type": "string", "format": "byte" }, - "piece_manager": { + "pieceManager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", "allOf": [ { @@ -929,13 +929,13 @@ const docTemplate = `{ } ] }, - "piece_cid": { + "pieceCid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", "type": "string", "format": "cid", "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" }, - "source_aggregate": { + "sourceAggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", "allOf": [ { @@ -943,7 +943,7 @@ const docTemplate = `{ } ] }, - "source_http": { + "sourceHttp": { "description": "SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.", "allOf": [ { @@ -951,7 +951,7 @@ const docTemplate = `{ } ] }, - "source_httpput": { + "sourceHttpPut": { "description": "SourceHttpPut allow clients to push piece data after deal is accepted", "allOf": [ { @@ -959,7 +959,7 @@ const docTemplate = `{ } ] }, - "source_offline": { + "sourceOffline": { "description": "SourceOffline defines the data source for offline pieces, including raw size information.", "allOf": [ { @@ -1069,7 +1069,7 @@ const docTemplate = `{ "mk20.DealProductStatusResponse": { "type": "object", "properties": { - "ddo_v1": { + "ddoV1": { "description": "DDOV1 holds the DealStatusResponse for product \"ddo_v1\".", "allOf": [ { @@ -1077,7 +1077,7 @@ const docTemplate = `{ } ] }, - "pdp_v1": { + "pdpV1": { "description": "PDPV1 represents the DealStatusResponse for the product pdp_v1.", "allOf": [ { @@ -1111,7 +1111,7 @@ const docTemplate = `{ "mk20.DealStatusResponse": { "type": "object", "properties": { - "error_msg": { + "errorMsg": { "description": "ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.", "type": "string" }, @@ -1181,34 +1181,34 @@ const docTemplate = `{ "mk20.PDPV1": { "type": "object", "properties": { - "add_piece": { + "addPiece": { "description": "AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined.", "type": "boolean" }, - "create_data_set": { + "createDataSet": { "description": "CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider.", "type": "boolean" }, - "data_set_id": { + "dataSetId": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", "type": "integer", "format": "uint64", "example": 0 }, - "delete_data_set": { + "deleteDataSet": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", "type": "boolean" }, - "delete_piece": { + "deletePiece": { "description": "DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined.", "type": "boolean" }, - "extra_data": { + "extraData": { "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", "type": "string", "format": "byte" }, - "piece_ids": { + "pieceIDs": { "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { @@ -1221,7 +1221,7 @@ const docTemplate = `{ 2 ] }, - "record_keeper": { + "recordKeeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", "type": "string" } @@ -1259,7 +1259,7 @@ const docTemplate = `{ "mk20.Products": { "type": "object", "properties": { - "ddo_v1": { + "ddoV1": { "description": "DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)", "allOf": [ { @@ -1267,7 +1267,7 @@ const docTemplate = `{ } ] }, - "pdp_v1": { + "pdpV1": { "description": "PDPV1 represents product-specific configuration for PDP version 1 deals.", "allOf": [ { @@ -1275,7 +1275,7 @@ const docTemplate = `{ } ] }, - "retrieval_v1": { + "retrievalV1": { "description": "RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags.", "allOf": [ { @@ -1288,11 +1288,11 @@ const docTemplate = `{ "mk20.RetrievalV1": { "type": "object", "properties": { - "announce_payload": { + "announcePayload": { "description": "AnnouncePayload indicates whether the payload should be announced to IPNI.", "type": "boolean" }, - "announce_piece": { + "announcePiece": { "description": "AnnouncePiece indicates whether the piece information should be announced to IPNI.", "type": "boolean" }, @@ -1305,11 +1305,11 @@ const docTemplate = `{ "mk20.StartUpload": { "type": "object", "properties": { - "chunk_size": { + "chunkSize": { "description": "ChunkSize defines the size of each data chunk to be used during the upload process.", "type": "integer" }, - "raw_size": { + "rawSize": { "description": "RawSize indicates the total size of the data to be uploaded in bytes.", "type": "integer" } @@ -1394,14 +1394,14 @@ const docTemplate = `{ "description": "Missing represents the number of chunks that are not yet uploaded.", "type": "integer" }, - "missing_chunks": { + "missingChunks": { "description": "MissingChunks is a slice containing the indices of missing chunks.", "type": "array", "items": { "type": "integer" } }, - "total_chunks": { + "totalChunks": { "description": "TotalChunks represents the total number of chunks required for the upload.", "type": "integer" }, @@ -1409,7 +1409,7 @@ const docTemplate = `{ "description": "Uploaded represents the number of chunks successfully uploaded.", "type": "integer" }, - "uploaded_chunks": { + "uploadedChunks": { "description": "UploadedChunks is a slice containing the indices of successfully uploaded chunks.", "type": "array", "items": { diff --git a/market/mk20/http/http.go b/market/mk20/http/http.go index 871eb39a2..2551b48d8 100644 --- a/market/mk20/http/http.go +++ b/market/mk20/http/http.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "embed" - _ "embed" "encoding/json" "errors" "fmt" @@ -220,7 +219,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { if r := recover(); r != nil { trace := make([]byte, 1<<16) n := runtime.Stack(trace, false) - log.Errorf("panic occurred: %v\n%s", r, trace[:n]) + log.Errorf("panic occurred in mk20deal: %v\n%s", r, trace[:n]) debug.PrintStack() } }() @@ -233,7 +232,10 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() + body, err := io.ReadAll(r.Body) if err != nil { log.Errorf("error reading request body: %s", err) @@ -250,10 +252,6 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { return } - log.Infof("DATA IS NULL = %t\n", deal.Data == nil) - - log.Infow("received deal proposal", "deal", deal) - authHeader := r.Header.Get("Authorization") if authHeader == "" { http.Error(w, "Missing Authorization header", http.StatusUnauthorized) @@ -268,7 +266,7 @@ func (mdh *MK20DealHandler) mk20deal(w http.ResponseWriter, r *http.Request) { "Reason", result.Reason) w.WriteHeader(int(result.HTTPCode)) - _, err = w.Write([]byte(fmt.Sprint("Reason: ", result.Reason))) + _, err = fmt.Fprint(w, "Reason: ", result.Reason) if err != nil { log.Errorw("writing deal response:", "id", deal.Identifier, "error", err) } @@ -612,7 +610,9 @@ func (mdh *MK20DealHandler) mk20FinalizeUpload(w http.ResponseWriter, r *http.Re http.Error(w, "error reading request body", http.StatusBadRequest) return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() log.Debugw("received upload finalize proposal", "id", idStr, "body", string(body)) @@ -693,7 +693,9 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() body, err := io.ReadAll(r.Body) if err != nil { @@ -725,7 +727,7 @@ func (mdh *MK20DealHandler) mk20UpdateDeal(w http.ResponseWriter, r *http.Reques "Reason", result.Reason) w.WriteHeader(int(result.HTTPCode)) - _, err = w.Write([]byte(fmt.Sprint("Reason: ", result.Reason))) + _, err = fmt.Fprint(w, "Reason: ", result.Reason) if err != nil { log.Errorw("writing deal update response:", "id", deal.Identifier, "error", err) } @@ -813,7 +815,9 @@ func (mdh *MK20DealHandler) mk20SerialUploadFinalize(w http.ResponseWriter, r *h http.Error(w, "error reading request body", http.StatusBadRequest) return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() log.Debugw("received serial upload finalize proposal", "id", idStr, "body", string(body)) diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 80ed8b02d..e3c0553bf 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -859,21 +859,21 @@ "mk20.DDOV1": { "type": "object", "properties": { - "allocation_id": { + "allocationId": { "description": "AllocationId represents an allocation identifier for the deal.", "type": "integer", "format": "uint64", "example": 1 }, - "contract_address": { + "contractAddress": { "description": "ContractAddress specifies the address of the contract governing the deal", "type": "string" }, - "contract_verify_method": { + "contractVerifyMethod": { "description": "ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract", "type": "string" }, - "contract_verify_method_params": { + "contractVerifyMethodParams": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", "type": "string", "format": "byte" @@ -882,16 +882,16 @@ "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", "type": "integer" }, - "notification_address": { + "notificationAddress": { "description": "NotificationAddress specifies the address to which notifications will be relayed to when sector is activated", "type": "string" }, - "notification_payload": { + "notificationPayload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", "type": "string", "format": "byte" }, - "piece_manager": { + "pieceManager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", "allOf": [ { @@ -920,13 +920,13 @@ } ] }, - "piece_cid": { + "pieceCid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", "type": "string", "format": "cid", "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" }, - "source_aggregate": { + "sourceAggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", "allOf": [ { @@ -934,7 +934,7 @@ } ] }, - "source_http": { + "sourceHttp": { "description": "SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval.", "allOf": [ { @@ -942,7 +942,7 @@ } ] }, - "source_httpput": { + "sourceHttpPut": { "description": "SourceHttpPut allow clients to push piece data after deal is accepted", "allOf": [ { @@ -950,7 +950,7 @@ } ] }, - "source_offline": { + "sourceOffline": { "description": "SourceOffline defines the data source for offline pieces, including raw size information.", "allOf": [ { @@ -1060,7 +1060,7 @@ "mk20.DealProductStatusResponse": { "type": "object", "properties": { - "ddo_v1": { + "ddoV1": { "description": "DDOV1 holds the DealStatusResponse for product \"ddo_v1\".", "allOf": [ { @@ -1068,7 +1068,7 @@ } ] }, - "pdp_v1": { + "pdpV1": { "description": "PDPV1 represents the DealStatusResponse for the product pdp_v1.", "allOf": [ { @@ -1102,7 +1102,7 @@ "mk20.DealStatusResponse": { "type": "object", "properties": { - "error_msg": { + "errorMsg": { "description": "ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred.", "type": "string" }, @@ -1172,34 +1172,34 @@ "mk20.PDPV1": { "type": "object", "properties": { - "add_piece": { + "addPiece": { "description": "AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined.", "type": "boolean" }, - "create_data_set": { + "createDataSet": { "description": "CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider.", "type": "boolean" }, - "data_set_id": { + "dataSetId": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", "type": "integer", "format": "uint64", "example": 0 }, - "delete_data_set": { + "deleteDataSet": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", "type": "boolean" }, - "delete_piece": { + "deletePiece": { "description": "DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined.", "type": "boolean" }, - "extra_data": { + "extraData": { "description": "ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed.", "type": "string", "format": "byte" }, - "piece_ids": { + "pieceIDs": { "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { @@ -1212,7 +1212,7 @@ 2 ] }, - "record_keeper": { + "recordKeeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", "type": "string" } @@ -1250,7 +1250,7 @@ "mk20.Products": { "type": "object", "properties": { - "ddo_v1": { + "ddoV1": { "description": "DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO)", "allOf": [ { @@ -1258,7 +1258,7 @@ } ] }, - "pdp_v1": { + "pdpV1": { "description": "PDPV1 represents product-specific configuration for PDP version 1 deals.", "allOf": [ { @@ -1266,7 +1266,7 @@ } ] }, - "retrieval_v1": { + "retrievalV1": { "description": "RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags.", "allOf": [ { @@ -1279,11 +1279,11 @@ "mk20.RetrievalV1": { "type": "object", "properties": { - "announce_payload": { + "announcePayload": { "description": "AnnouncePayload indicates whether the payload should be announced to IPNI.", "type": "boolean" }, - "announce_piece": { + "announcePiece": { "description": "AnnouncePiece indicates whether the piece information should be announced to IPNI.", "type": "boolean" }, @@ -1296,11 +1296,11 @@ "mk20.StartUpload": { "type": "object", "properties": { - "chunk_size": { + "chunkSize": { "description": "ChunkSize defines the size of each data chunk to be used during the upload process.", "type": "integer" }, - "raw_size": { + "rawSize": { "description": "RawSize indicates the total size of the data to be uploaded in bytes.", "type": "integer" } @@ -1385,14 +1385,14 @@ "description": "Missing represents the number of chunks that are not yet uploaded.", "type": "integer" }, - "missing_chunks": { + "missingChunks": { "description": "MissingChunks is a slice containing the indices of missing chunks.", "type": "array", "items": { "type": "integer" } }, - "total_chunks": { + "totalChunks": { "description": "TotalChunks represents the total number of chunks required for the upload.", "type": "integer" }, @@ -1400,7 +1400,7 @@ "description": "Uploaded represents the number of chunks successfully uploaded.", "type": "integer" }, - "uploaded_chunks": { + "uploadedChunks": { "description": "UploadedChunks is a slice containing the indices of successfully uploaded chunks.", "type": "array", "items": { diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index e8a9d2bbb..8c0daab96 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -17,20 +17,20 @@ definitions: - AggregateTypeV1 mk20.DDOV1: properties: - allocation_id: + allocationId: description: AllocationId represents an allocation identifier for the deal. example: 1 format: uint64 type: integer - contract_address: + contractAddress: description: ContractAddress specifies the address of the contract governing the deal type: string - contract_verify_method: + contractVerifyMethod: description: ContractDealIDMethod specifies the method name to verify the deal and retrieve the deal ID for a contract type: string - contract_verify_method_params: + contractVerifyMethodParams: description: ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract format: byte @@ -40,16 +40,16 @@ definitions: Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. It must be at least 518400 type: integer - notification_address: + notificationAddress: description: NotificationAddress specifies the address to which notifications will be relayed to when sector is activated type: string - notification_payload: + notificationPayload: description: NotificationPayload holds the notification data typically in a serialized byte array format. format: byte type: string - piece_manager: + pieceManager: allOf: - $ref: '#/definitions/address.Address' description: Actor providing AuthorizeMessage (like f1/f3 wallet) able to @@ -66,28 +66,28 @@ definitions: - $ref: '#/definitions/mk20.PieceDataFormat' description: Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. - piece_cid: + pieceCid: description: PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. example: bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq format: cid type: string - source_aggregate: + sourceAggregate: allOf: - $ref: '#/definitions/mk20.DataSourceAggregate' description: SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. - source_http: + sourceHttp: allOf: - $ref: '#/definitions/mk20.DataSourceHTTP' description: SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. - source_httpput: + sourceHttpPut: allOf: - $ref: '#/definitions/mk20.DataSourceHttpPut' description: SourceHttpPut allow clients to push piece data after deal is accepted - source_offline: + sourceOffline: allOf: - $ref: '#/definitions/mk20.DataSourceOffline' description: SourceOffline defines the data source for offline pieces, including @@ -169,11 +169,11 @@ definitions: - ErrDurationTooShort mk20.DealProductStatusResponse: properties: - ddo_v1: + ddoV1: allOf: - $ref: '#/definitions/mk20.DealStatusResponse' description: DDOV1 holds the DealStatusResponse for product "ddo_v1". - pdp_v1: + pdpV1: allOf: - $ref: '#/definitions/mk20.DealStatusResponse' description: PDPV1 represents the DealStatusResponse for the product pdp_v1. @@ -198,7 +198,7 @@ definitions: - DealStateComplete mk20.DealStatusResponse: properties: - error_msg: + errorMsg: description: ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. type: string @@ -249,35 +249,35 @@ definitions: type: object mk20.PDPV1: properties: - add_piece: + addPiece: description: AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined. type: boolean - create_data_set: + createDataSet: description: CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider. type: boolean - data_set_id: + dataSetId: description: DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. example: 0 format: uint64 type: integer - delete_data_set: + deleteDataSet: description: |- DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client. DataSetID must be defined. type: boolean - delete_piece: + deletePiece: description: DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined. type: boolean - extra_data: + extraData: description: ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. format: byte type: string - piece_ids: + pieceIDs: description: PieceIDs is a list of Piece ids in a proof set. example: - 0 @@ -287,7 +287,7 @@ definitions: format: uint64 type: integer type: array - record_keeper: + recordKeeper: description: RecordKeeper specifies the record keeper contract address for the new PDP dataset. type: string @@ -311,17 +311,17 @@ definitions: type: object mk20.Products: properties: - ddo_v1: + ddoV1: allOf: - $ref: '#/definitions/mk20.DDOV1' description: DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) - pdp_v1: + pdpV1: allOf: - $ref: '#/definitions/mk20.PDPV1' description: PDPV1 represents product-specific configuration for PDP version 1 deals. - retrieval_v1: + retrievalV1: allOf: - $ref: '#/definitions/mk20.RetrievalV1' description: RetrievalV1 represents configuration for retrieval settings in @@ -329,11 +329,11 @@ definitions: type: object mk20.RetrievalV1: properties: - announce_payload: + announcePayload: description: AnnouncePayload indicates whether the payload should be announced to IPNI. type: boolean - announce_piece: + announcePiece: description: AnnouncePiece indicates whether the piece information should be announced to IPNI. type: boolean @@ -344,11 +344,11 @@ definitions: type: object mk20.StartUpload: properties: - chunk_size: + chunkSize: description: ChunkSize defines the size of each data chunk to be used during the upload process. type: integer - raw_size: + rawSize: description: RawSize indicates the total size of the data to be uploaded in bytes. type: integer @@ -415,19 +415,19 @@ definitions: missing: description: Missing represents the number of chunks that are not yet uploaded. type: integer - missing_chunks: + missingChunks: description: MissingChunks is a slice containing the indices of missing chunks. items: type: integer type: array - total_chunks: + totalChunks: description: TotalChunks represents the total number of chunks required for the upload. type: integer uploaded: description: Uploaded represents the number of chunks successfully uploaded. type: integer - uploaded_chunks: + uploadedChunks: description: UploadedChunks is a slice containing the indices of successfully uploaded chunks. items: diff --git a/market/mk20/mk20.go b/market/mk20/mk20.go index d1d22950a..59e7337e6 100644 --- a/market/mk20/mk20.go +++ b/market/mk20/mk20.go @@ -257,7 +257,6 @@ func (m *MK20) processDDODeal(ctx context.Context, deal *Deal, tx *harmonydb.Tx) } func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRejectionInfo, error) { - fmt.Println("I HAVE ENTERED DDO SANITY CHECK") if !lo.Contains(m.miners, deal.Products.DDOV1.Provider) { return &ProviderDealRejectionInfo{ HTTPCode: ErrBadProposal, @@ -394,8 +393,6 @@ func (m *MK20) sanitizeDDODeal(ctx context.Context, deal *Deal) (*ProviderDealRe } } - fmt.Println("I HAVE EXITED DDO SANITY CHECK") - return nil, nil } @@ -644,8 +641,8 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error refIds = append(refIds, refID) } - n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) VALUES ($1, $2, $3, $4, $5)`, - dealID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1, refIds) + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) VALUES ($1, $2, $3, $4)`, + dealID, deal.Data.PieceCID.String(), ProductNamePDPV1, refIds) if err != nil { return xerrors.Errorf("inserting PDP download pipeline: %w", err) } @@ -672,10 +669,11 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error // Find all unique pieces where data source is HTTP type downloadkey struct { - ID string - PieceCID cid.Cid - Size abi.PaddedPieceSize - RawSize uint64 + ID string + PieceCIDV2 cid.Cid + PieceCID cid.Cid + Size abi.PaddedPieceSize + RawSize uint64 } toDownload := make(map[downloadkey][]HttpUrl) @@ -685,11 +683,11 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error return xerrors.Errorf("getting piece info: %w", err) } if piece.SourceHTTP != nil { - urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] + urls, ok := toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] if ok { - toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) + toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) } else { - toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs + toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs } } } @@ -721,15 +719,15 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error SELECT id, $4, $5, FALSE FROM selected_piece RETURNING ref_id ) - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) - VALUES ($6, $1, $2, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) - ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE + INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) + VALUES ($6, $8, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) + ON CONFLICT (id, piece_cid_v2, product) DO UPDATE SET ref_ids = array_append( market_mk20_download_pipeline.ref_ids, (SELECT ref_id FROM inserted_ref) ) WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, - k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, ProductNamePDPV1) + k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, ProductNamePDPV1, k.PieceCIDV2.String()) } if batch.Len() > batchSize { @@ -778,76 +776,82 @@ func insertPDPPipeline(ctx context.Context, tx *harmonydb.Tx, deal *Deal) error func markDownloaded(ctx context.Context, db *harmonydb.DB) { md := func(ctx context.Context, db *harmonydb.DB) { - var deals []struct { - ID string `db:"id"` - PieceCID string `db:"piece_cid_v2"` - } - - err := db.Select(ctx, &deals, `SELECT id, piece_cid_v2 FROM pdp_pipeline WHERE piece_ref IS NULL`) + //var deals []struct { + // ID string `db:"id"` + // PieceCID string `db:"piece_cid_v2"` + //} + // + //err := db.Select(ctx, &deals, `SELECT id, piece_cid_v2 FROM pdp_pipeline WHERE piece_ref IS NULL`) + //if err != nil { + // log.Errorw("error getting PDP deals", "error", err) + //} + // + //for _, deal := range deals { + // _, err = db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // pcid2, err := cid.Decode(deal.PieceCID) + // if err != nil { + // return false, xerrors.Errorf("decoding piece cid: %w", err) + // } + // + // pi, err := GetPieceInfo(pcid2) + // if err != nil { + // return false, xerrors.Errorf("getting piece info: %w", err) + // } + // + // var refid int64 + // err = tx.QueryRow(`SELECT u.ref_id FROM ( + // SELECT unnest(dp.ref_ids) AS ref_id + // FROM market_mk20_download_pipeline dp + // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + // ) u + // JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id + // JOIN parked_pieces pp ON pp.id = pr.piece_id + // WHERE pp.complete = TRUE + // LIMIT 1;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1).Scan(&refid) + // if err != nil { + // if errors.Is(err, pgx.ErrNoRows) { + // return false, nil + // } + // return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) + // } + // + // // Remove other ref_ids from piece_park_refs + // _, err = tx.Exec(`DELETE FROM parked_piece_refs + // WHERE ref_id IN ( + // SELECT unnest(dp.ref_ids) + // FROM market_mk20_download_pipeline dp + // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + // ) + // AND ref_id != $5;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1, refid) + // if err != nil { + // return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) + // } + // + // _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, + // deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1) + // if err != nil { + // return false, xerrors.Errorf("failed to delete piece from download table: %w", err) + // } + // + // _, err = tx.Exec(`UPDATE pdp_pipeline SET downloaded = TRUE, piece_ref = $1 + // WHERE id = $2 + // AND piece_cid_v2 = $3`, + // refid, deal.ID, deal.PieceCID) + // if err != nil { + // return false, xerrors.Errorf("failed to update download statos for PDP pipeline: %w", err) + // } + // return true, nil + // }, harmonydb.OptionRetry()) + // if err != nil { + // log.Errorw("error updating PDP deal", "deal", deal, "error", err) + // } + //} + n, err := db.Exec(ctx, `SELECT mk20_pdp_mark_downloaded($1)`, ProductNamePDPV1) if err != nil { - log.Errorw("error getting PDP deals", "error", err) - } - - for _, deal := range deals { - _, err = db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - pcid2, err := cid.Decode(deal.PieceCID) - if err != nil { - return false, xerrors.Errorf("decoding piece cid: %w", err) - } - - pi, err := GetPieceInfo(pcid2) - if err != nil { - return false, xerrors.Errorf("getting piece info: %w", err) - } - - var refid int64 - err = tx.QueryRow(`SELECT u.ref_id FROM ( - SELECT unnest(dp.ref_ids) AS ref_id - FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - ) u - JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id - JOIN parked_pieces pp ON pp.id = pr.piece_id - WHERE pp.complete = TRUE - LIMIT 1;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1).Scan(&refid) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return false, nil - } - return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) - } - - // Remove other ref_ids from piece_park_refs - _, err = tx.Exec(`DELETE FROM parked_piece_refs - WHERE ref_id IN ( - SELECT unnest(dp.ref_ids) - FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - ) - AND ref_id != $5;`, deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1, refid) - if err != nil { - return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) - } - - _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, - deal.ID, pi.PieceCIDV1.String(), pi.Size, ProductNamePDPV1) - if err != nil { - return false, xerrors.Errorf("failed to delete piece from download table: %w", err) - } - - _, err = tx.Exec(`UPDATE pdp_pipeline SET downloaded = TRUE, piece_ref = $1 - WHERE id = $2 - AND piece_cid_v2 = $3`, - refid, deal.ID, deal.PieceCID) - if err != nil { - return false, xerrors.Errorf("failed to update download statos for PDP pipeline: %w", err) - } - return true, nil - }, harmonydb.OptionRetry()) - if err != nil { - log.Errorw("error updating PDP deal", "deal", deal, "error", err) - } + log.Errorf("failed to mark PDP downloaded piece: %v", err) + return } + log.Debugf("Succesfully marked %d PDP pieces as downloaded", n) } ticker := time.NewTicker(time.Second * 2) diff --git a/market/mk20/mk20_upload.go b/market/mk20/mk20_upload.go index 9cb1952fd..baf4bf027 100644 --- a/market/mk20/mk20_upload.go +++ b/market/mk20/mk20_upload.go @@ -255,7 +255,9 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w } ctx := context.Background() - defer data.Close() + defer func() { + _ = data.Close() + }() if chunk < 1 { http.Error(w, "chunk must be greater than 0", int(UploadBadRequest)) @@ -308,7 +310,7 @@ func (m *MK20) HandleUploadChunk(id ulid.ULID, chunk int, data io.ReadCloser, w // Generate unique tmp pieceCID and Size for parked_pieces tables wr := new(commp.Calc) - n, err := wr.Write([]byte(fmt.Sprintf("%s, %d, %d, %s", id.String(), chunk, chunkSize, time.Now().String()))) + n, err := fmt.Fprintf(wr, "%s, %d, %d, %s", id.String(), chunk, chunkSize, time.Now().String()) if err != nil { log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "chunk", chunk, "error", err) http.Error(w, "", int(UploadServerError)) @@ -679,7 +681,7 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW // Generate unique tmp pieceCID and Size for parked_pieces tables wr := new(commp.Calc) - trs, err := wr.Write([]byte(fmt.Sprintf("%s, %s", id.String(), time.Now().String()))) + trs, err := fmt.Fprintf(wr, "%s, %s", id.String(), time.Now().String()) if err != nil { log.Errorw("failed to generate unique tmp pieceCID and Size for parked_pieces tables", "deal", id, "error", err) http.Error(w, "", int(UploadServerError)) @@ -888,7 +890,9 @@ func (m *MK20) HandleSerialUpload(id ulid.ULID, body io.Reader, w http.ResponseW return false, xerrors.Errorf("updating parked piece: expected 1 row updated, got %d", n) } } else { - defer pr.Close() + defer func() { + _ = pr.Close() + }() // Add parked_piece_ref if no errors var newRefID int64 err = tx.QueryRow(`INSERT INTO parked_piece_refs (piece_id, data_url, long_term) @@ -1070,8 +1074,6 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo return } - fmt.Println("I HAVE REACHED SANITY CHECK") - if uDeal.Products.DDOV1 != nil { rej, err := m.sanitizeDDODeal(ctx, uDeal) if err != nil { @@ -1093,8 +1095,6 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } } - fmt.Println("I HAVE FINISHED SANITIZING DDO DEAL") - if uDeal.Products.PDPV1 != nil { rej, err := m.sanitizePDPDeal(ctx, uDeal) if err != nil { @@ -1116,8 +1116,6 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } } - fmt.Println("I HAVE FINISHED SANITIZING PDP DEAL") - comm, err := m.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { _, err = tx.Exec(`DELETE FROM market_mk20_upload_waiting WHERE id = $1`, id.String()) if err != nil { @@ -1132,12 +1130,7 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo } } - fmt.Println("I HAVE FINISHED UPDATING DEAL") - retv := uDeal.Products.RetrievalV1 - if retv != nil { - fmt.Println("I HAVE RETRIEVAL V1") - } data := uDeal.Data aggregation := 0 @@ -1185,7 +1178,6 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo log.Debugw("mk20 pipeline created", "deal", id) refUsed = true - fmt.Println("I HAVE FINISHED CREATING MK20 PIPELINE") } if uDeal.Products.PDPV1 != nil { @@ -1203,8 +1195,8 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, data_set_id, - extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10)`, + extra_data, piece_ref, downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10, TRUE)`, id.String(), uDeal.Client, uDeal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, refID, aggregation, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { @@ -1214,7 +1206,6 @@ func (m *MK20) HandleSerialUploadFinalize(id ulid.ULID, deal *Deal, w http.Respo return false, xerrors.Errorf("inserting in PDP pipeline: %d rows affected", n) } log.Debugw("PDP pipeline created", "deal", id) - fmt.Println("I HAVE FINISHED CREATING PDP PIPELINE") } return true, nil diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 2abb99de6..db0cf8c77 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -13,29 +13,29 @@ import ( // PDPV1 represents configuration for product-specific PDP version 1 deals. type PDPV1 struct { // CreateDataSet indicated that this deal is meant to create a new DataSet for the client by storage provider. - CreateDataSet bool `json:"create_data_set"` + CreateDataSet bool `json:"createDataSet"` // DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client. // DataSetID must be defined. - DeleteDataSet bool `json:"delete_data_set"` + DeleteDataSet bool `json:"deleteDataSet"` // AddPiece indicated that this deal is meant to add Piece to a given DataSet. DataSetID must be defined. - AddPiece bool `json:"add_piece"` + AddPiece bool `json:"addPiece"` // DeletePiece indicates whether the Piece of the data should be deleted. DataSetID must be defined. - DeletePiece bool `json:"delete_piece"` + DeletePiece bool `json:"deletePiece"` // DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. - DataSetID *uint64 `json:"data_set_id,omitempty" swaggertype:"integer" format:"uint64" example:"0"` + DataSetID *uint64 `json:"dataSetId,omitempty" swaggertype:"integer" format:"uint64" example:"0"` // RecordKeeper specifies the record keeper contract address for the new PDP dataset. - RecordKeeper string `json:"record_keeper"` + RecordKeeper string `json:"recordKeeper"` // PieceIDs is a list of Piece ids in a proof set. - PieceIDs []uint64 `json:"piece_ids,omitempty" swaggertype:"array,integer" format:"uint64" example:"0,1,2"` + PieceIDs []uint64 `json:"pieceIDs,omitempty" swaggertype:"array,integer" format:"uint64" example:"0,1,2"` // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. - ExtraData []byte `json:"extra_data,omitempty" swaggertype:"string" format:"byte"` + ExtraData []byte `json:"extraData,omitempty" swaggertype:"string" format:"byte"` } func (p *PDPV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { diff --git a/market/mk20/retrieval_v1.go b/market/mk20/retrieval_v1.go index 5fa69ac9f..7749faf24 100644 --- a/market/mk20/retrieval_v1.go +++ b/market/mk20/retrieval_v1.go @@ -13,10 +13,10 @@ type RetrievalV1 struct { Indexing bool `json:"indexing"` // AnnouncePayload indicates whether the payload should be announced to IPNI. - AnnouncePayload bool `json:"announce_payload"` + AnnouncePayload bool `json:"announcePayload"` // AnnouncePiece indicates whether the piece information should be announced to IPNI. - AnnouncePiece bool `json:"announce_piece"` + AnnouncePiece bool `json:"announcePiece"` } func (r *RetrievalV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { diff --git a/market/mk20/types.go b/market/mk20/types.go index e6aef6bb7..9c8db23b1 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -28,35 +28,35 @@ type Deal struct { type Products struct { // DDOV1 represents a product v1 configuration for Direct Data Onboarding (DDO) - DDOV1 *DDOV1 `json:"ddo_v1,omitempty"` + DDOV1 *DDOV1 `json:"ddoV1,omitempty"` // RetrievalV1 represents configuration for retrieval settings in the system, including indexing and announcement flags. - RetrievalV1 *RetrievalV1 `json:"retrieval_v1,omitempty"` + RetrievalV1 *RetrievalV1 `json:"retrievalV1,omitempty"` // PDPV1 represents product-specific configuration for PDP version 1 deals. - PDPV1 *PDPV1 `json:"pdp_v1,omitempty"` + PDPV1 *PDPV1 `json:"pdpV1,omitempty"` } // DataSource represents the source of piece data, including metadata and optional methods to fetch or describe the data origin. type DataSource struct { // PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. - PieceCID cid.Cid `json:"piece_cid" swaggertype:"string" format:"cid" example:"bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"` + PieceCID cid.Cid `json:"pieceCid" swaggertype:"string" format:"cid" example:"bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"` // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` // SourceHTTP represents the HTTP-based source of piece data within a deal, including raw size and URLs for retrieval. - SourceHTTP *DataSourceHTTP `json:"source_http,omitempty"` + SourceHTTP *DataSourceHTTP `json:"sourceHttp,omitempty"` // SourceAggregate represents an aggregated source, comprising multiple data sources as pieces. - SourceAggregate *DataSourceAggregate `json:"source_aggregate,omitempty"` + SourceAggregate *DataSourceAggregate `json:"sourceAggregate,omitempty"` // SourceOffline defines the data source for offline pieces, including raw size information. - SourceOffline *DataSourceOffline `json:"source_offline,omitempty"` + SourceOffline *DataSourceOffline `json:"sourceOffline,omitempty"` // SourceHttpPut allow clients to push piece data after deal is accepted - SourceHttpPut *DataSourceHttpPut `json:"source_httpput,omitempty"` + SourceHttpPut *DataSourceHttpPut `json:"sourceHttpPut,omitempty"` } // PieceDataFormat represents various formats in which piece data can be defined, including CAR files, aggregate formats, or raw byte data. diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go new file mode 100644 index 000000000..150bf4093 --- /dev/null +++ b/market/mk20/types_test.go @@ -0,0 +1,297 @@ +package mk20 + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + "testing" + + "github.com/ipfs/go-cid" + "github.com/oklog/ulid" + "github.com/stretchr/testify/require" +) + +func mustCID(t *testing.T, s string) cid.Cid { + t.Helper() + c, err := cid.Parse(s) + if err != nil { + t.Fatalf("parse cid: %v", err) + } + return c +} + +func mustULID(t *testing.T, s string) ulid.ULID { + t.Helper() + id, err := ulid.Parse(s) + if err != nil { + t.Fatalf("parse ulid: %v", err) + } + return id +} + +func TestDeal_MarshalUnmarshal_Minimal(t *testing.T) { + orig := Deal{ + Identifier: mustULID(t, "01ARZ3NDEKTSV4RRFFQ69G5FAV"), + Client: "f1abcclient", + // Data omitted (omitempty) + // Products is empty struct; inner fields are omitempty + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + // Expect "data" to be absent and "products" to be an empty object + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal into map: %v", err) + } + if _, ok := m["data"]; ok { + t.Fatalf("expected 'data' to be omitted, found present") + } + if p, ok := m["products"]; !ok { + t.Fatalf("expected 'products' present") + } else if obj, ok := p.(map[string]any); !ok || len(obj) != 0 { + t.Fatalf("expected 'products' to be empty object, got: %#v", p) + } + + var round Deal + if err := json.Unmarshal(b, &round); err != nil { + t.Fatalf("round unmarshal: %v", err) + } + + if !reflect.DeepEqual(orig, round) { + t.Fatalf("round trip mismatch:\norig: %#v\nround:%#v", orig, round) + } +} + +func TestHttpHeaderRoundTrip(t *testing.T) { + orig := http.Header{ + "X-Trace-Id": []string{"abc123"}, + "Cache-Control": []string{"no-cache", "private"}, + } + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + t.Logf("marshaled JSON: %s", string(b)) + + var round http.Header + if err := json.Unmarshal(b, &round); err != nil { + t.Fatalf("unmarshal: %v", err) + } + t.Logf("unmarshaled Struct: %+v", round) + v := round.Values("Cache-Control") + require.Equal(t, 2, len(v)) + require.Equal(t, "no-cache", v[0]) + require.Equal(t, "private", v[1]) + v = round.Values("X-Trace-Id") + require.Equal(t, 1, len(v)) + require.Equal(t, "abc123", v[0]) +} + +func TestDeal_HTTPSourceWithHeaders(t *testing.T) { + orig := Deal{ + Identifier: mustULID(t, "01ARZ3NDEKTSV4RRFFQ69G5FAV"), + Client: "f1client", + Data: &DataSource{ + PieceCID: mustCID(t, "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"), + Format: PieceDataFormat{Car: &FormatCar{}}, + SourceHTTP: &DataSourceHTTP{ + URLs: []HttpUrl{ + { + URL: "https://example.com/piece/xyz", + Headers: http.Header{"X-Trace-Id": []string{"abc123"}, "Cache-Control": []string{"no-cache", "private"}}, + Priority: 10, + Fallback: false, + }, + { + URL: "http://127.0.0.1:8080/piece/xyz", + Headers: http.Header{}, // empty headers should round-trip + Priority: 20, + Fallback: true, + }, + }, + }, + }, + Products: Products{}, // explicit empty + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var round Deal + if err := json.Unmarshal(b, &round); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if !reflect.DeepEqual(orig, round) { + t.Fatalf("round trip mismatch:\norig: %#v\nround:%#v", orig, round) + } + + // Spot-check headers survived correctly + gotHdr := round.Data.SourceHTTP.URLs[0].Headers + if v := gotHdr.Get("X-Trace-ID"); v != "abc123" { + t.Fatalf("expected X-Trace-ID=abc123, got %q", v) + } +} + +func TestDeal_Aggregate_NoSub_vs_EmptySub(t *testing.T) { + // Case A: Aggregate.Sub is nil (no omitempty on Sub), expected to marshal as "sub": null + withNil := Deal{ + Identifier: mustULID(t, "01ARZ3NDEKTSV4RRFFQ69G5FAV"), + Client: "f1client", + Data: &DataSource{ + PieceCID: mustCID(t, "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"), + Format: PieceDataFormat{ + Aggregate: &FormatAggregate{ + Type: AggregateTypeV1, + Sub: nil, // important + }, + }, + }, + } + + bNil, err := json.Marshal(withNil) + if err != nil { + t.Fatalf("marshal nil-sub: %v", err) + } + var objNil map[string]any + _ = json.Unmarshal(bNil, &objNil) // ignore error; presence check is all we need + // Navigate: data.format.aggregate.sub should be null + dataMap := objNil["data"].(map[string]any) + format := dataMap["format"].(map[string]any) + agg := format["aggregate"].(map[string]any) + if _, ok := agg["sub"]; !ok { + t.Fatalf("expected aggregate.sub to be present (as null) when Sub == nil") + } + if agg["sub"] != nil { + t.Fatalf("expected aggregate.sub == null; got: %#v", agg["sub"]) + } + + // Case B: Aggregate.Sub is empty slice, expected to marshal as "sub": [] + withEmpty := withNil + withEmpty.Data.Format.Aggregate.Sub = []DataSource{} + + bEmpty, err := json.Marshal(withEmpty) + if err != nil { + t.Fatalf("marshal empty-sub: %v", err) + } + var objEmpty map[string]any + _ = json.Unmarshal(bEmpty, &objEmpty) + dataMap = objEmpty["data"].(map[string]any) + format = dataMap["format"].(map[string]any) + agg = format["aggregate"].(map[string]any) + arr, ok := agg["sub"].([]any) + if !ok { + t.Fatalf("expected aggregate.sub to be [] when Sub == empty slice; got %#v", agg["sub"]) + } + if len(arr) != 0 { + t.Fatalf("expected empty array for sub; got len=%d", len(arr)) + } +} + +func TestDeal_Aggregate_WithSubpieces_RoundTrip(t *testing.T) { + // Two subpieces: one Raw, one Car + sub1 := DataSource{ + PieceCID: mustCID(t, "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"), + Format: PieceDataFormat{Raw: &FormatBytes{}}, + SourceOffline: &DataSourceOffline{}, // ensure additional fields survive + } + sub2 := DataSource{ + PieceCID: mustCID(t, "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pd"), + Format: PieceDataFormat{Car: &FormatCar{}}, + } + + orig := Deal{ + Identifier: mustULID(t, "01ARZ3NDEKTSV4RRFFQ69G5FAV"), + Client: "f1client", + Data: &DataSource{ + PieceCID: mustCID(t, "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pe"), + Format: PieceDataFormat{ + Aggregate: &FormatAggregate{ + Type: AggregateTypeV1, + Sub: []DataSource{sub1, sub2}, + }, + }, + SourceAggregate: &DataSourceAggregate{Pieces: []DataSource{sub1, sub2}}, + }, + Products: Products{ + // exercise omitempty pointers: all nil + }, + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var round Deal + if err := json.Unmarshal(b, &round); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + // Order must be preserved + if len(round.Data.Format.Aggregate.Sub) != 2 { + t.Fatalf("expected 2 subpieces, got %d", len(round.Data.Format.Aggregate.Sub)) + } + if round.Data.Format.Aggregate.Sub[0].PieceCID.String() != sub1.PieceCID.String() { + t.Fatalf("subpiece[0] order changed") + } + + if !reflect.DeepEqual(orig, round) { + t.Fatalf("round trip mismatch:\norig: %#v\nround:%#v", orig, round) + } +} + +func TestDeal_Products_OmitEmptyInnerFields(t *testing.T) { + // All product pointers nil -> products should marshal as {} + orig := Deal{ + Identifier: mustULID(t, "01ARZ3NDEKTSV4RRFFQ69G5FAV"), + Client: "f1client", + Products: Products{}, + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("unmarshal map: %v", err) + } + p, ok := m["products"] + if !ok { + t.Fatalf("products missing") + } + if obj, ok := p.(map[string]any); !ok || len(obj) != 0 { + t.Fatalf("expected products to be {}, got %#v", p) + } + + var round Deal + if err := json.Unmarshal(b, &round); err != nil { + t.Fatalf("round unmarshal: %v", err) + } + if !reflect.DeepEqual(orig.Products, round.Products) { + t.Fatalf("products changed on round trip: %#v -> %#v", orig.Products, round.Products) + } +} + +func TestManualUnMarshal(t *testing.T) { + iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdp_v1\":{\"create_data_Set\":true,\"add_piece\":true,\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" + var deal Deal + if err := json.Unmarshal([]byte(iString), &deal); err != nil { + t.Fatal(err) + } + fmt.Printf("%+v\n", deal) + require.NotNil(t, deal) + require.NotNil(t, deal.Products) + require.NotNil(t, deal.Products.PDPV1) + require.Equal(t, true, deal.Products.PDPV1.CreateDataSet) + require.Equal(t, true, deal.Products.PDPV1.AddPiece) + require.Equal(t, "0x158c8f05A616403589b99BE5d82d756860363A92", deal.Products.PDPV1.RecordKeeper) +} diff --git a/market/mk20/utils.go b/market/mk20/utils.go index 51356e0c9..f17f3b916 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -674,17 +674,17 @@ type DealStatusResponse struct { State DealState `json:"status"` // ErrorMsg is an optional field containing error details associated with the deal's current state if an error occurred. - ErrorMsg string `json:"error_msg"` + ErrorMsg string `json:"errorMsg"` } // DealProductStatusResponse represents the status response for deal products with their respective deal statuses. type DealProductStatusResponse struct { // DDOV1 holds the DealStatusResponse for product "ddo_v1". - DDOV1 DealStatusResponse `json:"ddo_v1"` + DDOV1 DealStatusResponse `json:"ddoV1"` // PDPV1 represents the DealStatusResponse for the product pdp_v1. - PDPV1 DealStatusResponse `json:"pdp_v1"` + PDPV1 DealStatusResponse `json:"pdpV1"` } // DealStatus represents the status of a deal, including the HTTP code and an optional response detailing the deal's state and error message. @@ -797,17 +797,17 @@ type SupportedDataSources struct { type StartUpload struct { // RawSize indicates the total size of the data to be uploaded in bytes. - RawSize uint64 `json:"raw_size"` + RawSize uint64 `json:"rawSize"` // ChunkSize defines the size of each data chunk to be used during the upload process. - ChunkSize int64 `json:"chunk_size"` + ChunkSize int64 `json:"chunkSize"` } // UploadStatus represents the status of a file upload process, including progress and missing chunks. type UploadStatus struct { // TotalChunks represents the total number of chunks required for the upload. - TotalChunks int `json:"total_chunks"` + TotalChunks int `json:"totalChunks"` // Uploaded represents the number of chunks successfully uploaded. Uploaded int `json:"uploaded"` @@ -816,10 +816,10 @@ type UploadStatus struct { Missing int `json:"missing"` // UploadedChunks is a slice containing the indices of successfully uploaded chunks. - UploadedChunks []int `json:"uploaded_chunks"` + UploadedChunks []int `json:"uploadedChunks"` //MissingChunks is a slice containing the indices of missing chunks. - MissingChunks []int `json:"missing_chunks"` + MissingChunks []int `json:"missingChunks"` } func UpdateDealDetails(ctx context.Context, db *harmonydb.DB, id ulid.ULID, deal *Deal, cfg *config.MK20Config, auth string) (*Deal, DealCode, []ProductName, error) { diff --git a/market/retrieval/piecehandler.go b/market/retrieval/piecehandler.go index 1d1db2022..fd5d6b0cf 100644 --- a/market/retrieval/piecehandler.go +++ b/market/retrieval/piecehandler.go @@ -62,7 +62,7 @@ func (rp *Provider) handleByPieceCid(w http.ResponseWriter, r *http.Request) { reader, size, err := rp.cpr.GetSharedPieceReader(ctx, pieceCid) if err != nil { log.Errorf("server error getting content for piece CID %s: %s", pieceCid, err) - if errors.Is(err, cachedreader.NoDealErr) { + if errors.Is(err, cachedreader.ErrNoDeal) { w.WriteHeader(http.StatusNotFound) stats.Record(ctx, remoteblockstore.HttpPieceByCid404ResponseCount.M(1)) return diff --git a/market/storageingest/deal_ingest_seal.go b/market/storageingest/deal_ingest_seal.go index ceb59d211..7c62cd4bd 100644 --- a/market/storageingest/deal_ingest_seal.go +++ b/market/storageingest/deal_ingest_seal.go @@ -311,7 +311,7 @@ func (p *PieceIngester) AllocatePieceToSector(ctx context.Context, tx *harmonydb } // Reject incorrect sized online deals except verified deal less than 1 MiB because verified deals can be 1 MiB minimum even if rawSize is much lower - if psize != padreader.PaddedSize(uint64(rawSize)).Padded() && !(vd.isVerified && psize <= abi.PaddedPieceSize(1<<20)) { + if psize != padreader.PaddedSize(uint64(rawSize)).Padded() && (!vd.isVerified || psize > abi.PaddedPieceSize(1<<20)) { return nil, nil, xerrors.Errorf("raw size doesn't match padded piece size") } diff --git a/market/storageingest/deal_ingest_snap.go b/market/storageingest/deal_ingest_snap.go index 2d4cdd8eb..d271f8a0c 100644 --- a/market/storageingest/deal_ingest_snap.go +++ b/market/storageingest/deal_ingest_snap.go @@ -301,7 +301,7 @@ func (p *PieceIngesterSnap) AllocatePieceToSector(ctx context.Context, tx *harmo } // Reject incorrect sized online deals except verified deal less than 1 MiB because verified deals can be 1 MiB minimum even if rawSize is much lower - if psize != padreader.PaddedSize(uint64(rawSize)).Padded() && !(vd.isVerified && psize <= abi.PaddedPieceSize(1<<20)) { + if psize != padreader.PaddedSize(uint64(rawSize)).Padded() && (!vd.isVerified || psize > abi.PaddedPieceSize(1<<20)) { return nil, nil, xerrors.Errorf("raw size doesn't match padded piece size") } diff --git a/pdp/contract/ListenerServiceWithViewContract.abi b/pdp/contract/ListenerServiceWithViewContract.abi new file mode 100644 index 000000000..5040222da --- /dev/null +++ b/pdp/contract/ListenerServiceWithViewContract.abi @@ -0,0 +1,15 @@ +[ + { + "type": "function", + "name": "viewContractAddress", + "inputs": [], + "outputs": [ + { + "name": "", + "type": "address", + "internalType": "address" + } + ], + "stateMutability": "view" + } +] \ No newline at end of file diff --git a/pdp/contract/ListenerServiceWithViewContract.go b/pdp/contract/ListenerServiceWithViewContract.go new file mode 100644 index 000000000..44ec65320 --- /dev/null +++ b/pdp/contract/ListenerServiceWithViewContract.go @@ -0,0 +1,212 @@ +// Code generated - DO NOT EDIT. +// This file is a generated binding and any manual changes will be lost. + +package contract + +import ( + "errors" + "math/big" + "strings" + + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" +) + +// Reference imports to suppress errors if they are not otherwise used. +var ( + _ = errors.New + _ = big.NewInt + _ = strings.NewReader + _ = ethereum.NotFound + _ = bind.Bind + _ = common.Big1 + _ = types.BloomLookup + _ = event.NewSubscription + _ = abi.ConvertType +) + +// ListenerServiceWithViewContractMetaData contains all meta data concerning the ListenerServiceWithViewContract contract. +var ListenerServiceWithViewContractMetaData = &bind.MetaData{ + ABI: "[{\"type\":\"function\",\"name\":\"viewContractAddress\",\"inputs\":[],\"outputs\":[{\"name\":\"\",\"type\":\"address\",\"internalType\":\"address\"}],\"stateMutability\":\"view\"}]", +} + +// ListenerServiceWithViewContractABI is the input ABI used to generate the binding from. +// Deprecated: Use ListenerServiceWithViewContractMetaData.ABI instead. +var ListenerServiceWithViewContractABI = ListenerServiceWithViewContractMetaData.ABI + +// ListenerServiceWithViewContract is an auto generated Go binding around an Ethereum contract. +type ListenerServiceWithViewContract struct { + ListenerServiceWithViewContractCaller // Read-only binding to the contract + ListenerServiceWithViewContractTransactor // Write-only binding to the contract + ListenerServiceWithViewContractFilterer // Log filterer for contract events +} + +// ListenerServiceWithViewContractCaller is an auto generated read-only Go binding around an Ethereum contract. +type ListenerServiceWithViewContractCaller struct { + contract *bind.BoundContract // Generic contract wrapper for the low level calls +} + +// ListenerServiceWithViewContractTransactor is an auto generated write-only Go binding around an Ethereum contract. +type ListenerServiceWithViewContractTransactor struct { + contract *bind.BoundContract // Generic contract wrapper for the low level calls +} + +// ListenerServiceWithViewContractFilterer is an auto generated log filtering Go binding around an Ethereum contract events. +type ListenerServiceWithViewContractFilterer struct { + contract *bind.BoundContract // Generic contract wrapper for the low level calls +} + +// ListenerServiceWithViewContractSession is an auto generated Go binding around an Ethereum contract, +// with pre-set call and transact options. +type ListenerServiceWithViewContractSession struct { + Contract *ListenerServiceWithViewContract // Generic contract binding to set the session for + CallOpts bind.CallOpts // Call options to use throughout this session + TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session +} + +// ListenerServiceWithViewContractCallerSession is an auto generated read-only Go binding around an Ethereum contract, +// with pre-set call options. +type ListenerServiceWithViewContractCallerSession struct { + Contract *ListenerServiceWithViewContractCaller // Generic contract caller binding to set the session for + CallOpts bind.CallOpts // Call options to use throughout this session +} + +// ListenerServiceWithViewContractTransactorSession is an auto generated write-only Go binding around an Ethereum contract, +// with pre-set transact options. +type ListenerServiceWithViewContractTransactorSession struct { + Contract *ListenerServiceWithViewContractTransactor // Generic contract transactor binding to set the session for + TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session +} + +// ListenerServiceWithViewContractRaw is an auto generated low-level Go binding around an Ethereum contract. +type ListenerServiceWithViewContractRaw struct { + Contract *ListenerServiceWithViewContract // Generic contract binding to access the raw methods on +} + +// ListenerServiceWithViewContractCallerRaw is an auto generated low-level read-only Go binding around an Ethereum contract. +type ListenerServiceWithViewContractCallerRaw struct { + Contract *ListenerServiceWithViewContractCaller // Generic read-only contract binding to access the raw methods on +} + +// ListenerServiceWithViewContractTransactorRaw is an auto generated low-level write-only Go binding around an Ethereum contract. +type ListenerServiceWithViewContractTransactorRaw struct { + Contract *ListenerServiceWithViewContractTransactor // Generic write-only contract binding to access the raw methods on +} + +// NewListenerServiceWithViewContract creates a new instance of ListenerServiceWithViewContract, bound to a specific deployed contract. +func NewListenerServiceWithViewContract(address common.Address, backend bind.ContractBackend) (*ListenerServiceWithViewContract, error) { + contract, err := bindListenerServiceWithViewContract(address, backend, backend, backend) + if err != nil { + return nil, err + } + return &ListenerServiceWithViewContract{ListenerServiceWithViewContractCaller: ListenerServiceWithViewContractCaller{contract: contract}, ListenerServiceWithViewContractTransactor: ListenerServiceWithViewContractTransactor{contract: contract}, ListenerServiceWithViewContractFilterer: ListenerServiceWithViewContractFilterer{contract: contract}}, nil +} + +// NewListenerServiceWithViewContractCaller creates a new read-only instance of ListenerServiceWithViewContract, bound to a specific deployed contract. +func NewListenerServiceWithViewContractCaller(address common.Address, caller bind.ContractCaller) (*ListenerServiceWithViewContractCaller, error) { + contract, err := bindListenerServiceWithViewContract(address, caller, nil, nil) + if err != nil { + return nil, err + } + return &ListenerServiceWithViewContractCaller{contract: contract}, nil +} + +// NewListenerServiceWithViewContractTransactor creates a new write-only instance of ListenerServiceWithViewContract, bound to a specific deployed contract. +func NewListenerServiceWithViewContractTransactor(address common.Address, transactor bind.ContractTransactor) (*ListenerServiceWithViewContractTransactor, error) { + contract, err := bindListenerServiceWithViewContract(address, nil, transactor, nil) + if err != nil { + return nil, err + } + return &ListenerServiceWithViewContractTransactor{contract: contract}, nil +} + +// NewListenerServiceWithViewContractFilterer creates a new log filterer instance of ListenerServiceWithViewContract, bound to a specific deployed contract. +func NewListenerServiceWithViewContractFilterer(address common.Address, filterer bind.ContractFilterer) (*ListenerServiceWithViewContractFilterer, error) { + contract, err := bindListenerServiceWithViewContract(address, nil, nil, filterer) + if err != nil { + return nil, err + } + return &ListenerServiceWithViewContractFilterer{contract: contract}, nil +} + +// bindListenerServiceWithViewContract binds a generic wrapper to an already deployed contract. +func bindListenerServiceWithViewContract(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) { + parsed, err := ListenerServiceWithViewContractMetaData.GetAbi() + if err != nil { + return nil, err + } + return bind.NewBoundContract(address, *parsed, caller, transactor, filterer), nil +} + +// Call invokes the (constant) contract method with params as input values and +// sets the output to result. The result type might be a single field for simple +// returns, a slice of interfaces for anonymous returns and a struct for named +// returns. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractRaw) Call(opts *bind.CallOpts, result *[]interface{}, method string, params ...interface{}) error { + return _ListenerServiceWithViewContract.Contract.ListenerServiceWithViewContractCaller.contract.Call(opts, result, method, params...) +} + +// Transfer initiates a plain transaction to move funds to the contract, calling +// its default method if one is available. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { + return _ListenerServiceWithViewContract.Contract.ListenerServiceWithViewContractTransactor.contract.Transfer(opts) +} + +// Transact invokes the (paid) contract method with params as input values. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { + return _ListenerServiceWithViewContract.Contract.ListenerServiceWithViewContractTransactor.contract.Transact(opts, method, params...) +} + +// Call invokes the (constant) contract method with params as input values and +// sets the output to result. The result type might be a single field for simple +// returns, a slice of interfaces for anonymous returns and a struct for named +// returns. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractCallerRaw) Call(opts *bind.CallOpts, result *[]interface{}, method string, params ...interface{}) error { + return _ListenerServiceWithViewContract.Contract.contract.Call(opts, result, method, params...) +} + +// Transfer initiates a plain transaction to move funds to the contract, calling +// its default method if one is available. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) { + return _ListenerServiceWithViewContract.Contract.contract.Transfer(opts) +} + +// Transact invokes the (paid) contract method with params as input values. +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) { + return _ListenerServiceWithViewContract.Contract.contract.Transact(opts, method, params...) +} + +// ViewContractAddress is a free data retrieval call binding the contract method 0x7a9ebc15. +// +// Solidity: function viewContractAddress() view returns(address) +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractCaller) ViewContractAddress(opts *bind.CallOpts) (common.Address, error) { + var out []interface{} + err := _ListenerServiceWithViewContract.contract.Call(opts, &out, "viewContractAddress") + + if err != nil { + return *new(common.Address), err + } + + out0 := *abi.ConvertType(out[0], new(common.Address)).(*common.Address) + + return out0, err + +} + +// ViewContractAddress is a free data retrieval call binding the contract method 0x7a9ebc15. +// +// Solidity: function viewContractAddress() view returns(address) +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractSession) ViewContractAddress() (common.Address, error) { + return _ListenerServiceWithViewContract.Contract.ViewContractAddress(&_ListenerServiceWithViewContract.CallOpts) +} + +// ViewContractAddress is a free data retrieval call binding the contract method 0x7a9ebc15. +// +// Solidity: function viewContractAddress() view returns(address) +func (_ListenerServiceWithViewContract *ListenerServiceWithViewContractCallerSession) ViewContractAddress() (common.Address, error) { + return _ListenerServiceWithViewContract.Contract.ViewContractAddress(&_ListenerServiceWithViewContract.CallOpts) +} diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index 1d2f36d1d..1447aefc3 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -13,7 +13,7 @@ import ( const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" -const PDPTestNet = "Change ME" +const PDPTestNet = "0xAa67E78b48ca16c8Ee2e9f296404637307D3654d" type PDPContracts struct { PDPVerifier common.Address diff --git a/pdp/contract/utils.go b/pdp/contract/utils.go new file mode 100644 index 000000000..72167f2bf --- /dev/null +++ b/pdp/contract/utils.go @@ -0,0 +1,37 @@ +package contract + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "golang.org/x/xerrors" +) + +// GetProvingScheduleFromListener checks if a listener has a view contract and returns +// an IPDPProvingSchedule instance bound to the appropriate address. +// It uses the view contract address if available, otherwise uses the listener address directly. +func GetProvingScheduleFromListener(listenerAddr common.Address, ethClient *ethclient.Client) (*IPDPProvingSchedule, error) { + // Try to get the view contract address from the listener + provingScheduleAddr := listenerAddr + + // Check if the listener supports the viewContractAddress method + listenerService, err := NewListenerServiceWithViewContract(listenerAddr, ethClient) + if err == nil { + // Try to get the view contract address + viewAddr, err := listenerService.ViewContractAddress(nil) + if err == nil && viewAddr != (common.Address{}) { + // Use the view contract for proving schedule operations + provingScheduleAddr = viewAddr + } + } + + // Create and return the IPDPProvingSchedule binding + // This works whether provingScheduleAddr points to: + // - The view contract (which must implement IPDPProvingSchedule) + // - The listener itself (where listener must implement IPDPProvingSchedule) + provingSchedule, err := NewIPDPProvingSchedule(provingScheduleAddr, ethClient) + if err != nil { + return nil, xerrors.Errorf("failed to create proving schedule binding: %w", err) + } + + return provingSchedule, nil +} diff --git a/pdp/handlers.go b/pdp/handlers.go index 5d183dce5..1ae8229c6 100644 --- a/pdp/handlers.go +++ b/pdp/handlers.go @@ -154,7 +154,9 @@ func (p *PDPService) handleCreateProofSet(w http.ResponseWriter, r *http.Request http.Error(w, "Failed to read request body: "+err.Error(), http.StatusBadRequest) return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() var reqBody RequestBody if err := json.Unmarshal(body, &reqBody); err != nil { @@ -358,7 +360,7 @@ func (p *PDPService) handleGetProofSetCreationStatus(w http.ResponseWriter, r *h WHERE create_message_hash = $1 `, txHash).Scan(&proofSetCreate.CreateMessageHash, &proofSetCreate.OK, &proofSetCreate.ProofSetCreated, &proofSetCreate.Service) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { http.Error(w, "Proof set creation not found for given txHash", http.StatusNotFound) return } @@ -395,7 +397,7 @@ func (p *PDPService) handleGetProofSetCreationStatus(w http.ResponseWriter, r *h WHERE signed_tx_hash = $1 `, txHash).Scan(&txStatus) if err != nil { - if err == sql.ErrNoRows { + if errors.Is(err, pgx.ErrNoRows) { // This should not happen as per foreign key constraints http.Error(w, "Message status not found for given txHash", http.StatusInternalServerError) return @@ -648,7 +650,9 @@ func (p *PDPService) handleAddRootToProofSet(w http.ResponseWriter, r *http.Requ http.Error(w, "Invalid request body: "+err.Error(), http.StatusBadRequest) return } - defer r.Body.Close() + defer func() { + _ = r.Body.Close() + }() if len(payload.Roots) == 0 { http.Error(w, "At least one root must be provided", http.StatusBadRequest) diff --git a/pdp/handlers_upload.go b/pdp/handlers_upload.go index 1f622a85e..69efc0840 100644 --- a/pdp/handlers_upload.go +++ b/pdp/handlers_upload.go @@ -202,7 +202,7 @@ func (p *PDPService) handlePiecePost(w http.ResponseWriter, r *http.Request) { VALUES ($1, $2, $3, $4, $5, $6, $7) `, uploadUUID.String(), serviceID, pieceCidStr, req.Notify, req.Check.Name, must.One(hex.DecodeString(req.Check.Hash)), req.Check.Size) if err != nil { - return false, fmt.Errorf("Failed to store upload request in database: %w", err) + return false, fmt.Errorf("failed to store upload request in database: %w", err) } // Create a location URL where the piece data can be uploaded via PUT @@ -217,16 +217,17 @@ func (p *PDPService) handlePiecePost(w http.ResponseWriter, r *http.Request) { return } - if responseStatus == http.StatusCreated { + switch responseStatus { + case http.StatusCreated: // Return 201 Created with Location header w.Header().Set("Location", uploadURL) w.WriteHeader(http.StatusCreated) - } else if responseStatus == http.StatusOK { + case http.StatusOK: // Return 200 OK with the pieceCID w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(map[string]string{"pieceCID": pieceCid.String()}) - } else { + default: // Should not reach here http.Error(w, "Unexpected error", http.StatusInternalServerError) } diff --git a/tasks/gc/storage_gc_mark.go b/tasks/gc/storage_gc_mark.go index 25777b819..4b9bbcc5a 100644 --- a/tasks/gc/storage_gc_mark.go +++ b/tasks/gc/storage_gc_mark.go @@ -218,7 +218,7 @@ func (s *StorageGCMark) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d if len(toRemove) > 0 { // persist new removal candidates for storageId, decls := range storageSectors { for _, decl := range decls { - for _, filetype := range decl.SectorFileType.AllSet() { + for _, filetype := range decl.AllSet() { if filetype == storiface.FTPiece { continue } @@ -405,7 +405,7 @@ func (s *StorageGCMark) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d for storageId, decls := range storageSectors { for _, decl := range decls { - if !decl.SectorFileType.Has(storiface.FTSealed) { + if !decl.Has(storiface.FTSealed) { continue } diff --git a/tasks/indexing/task_indexing.go b/tasks/indexing/task_indexing.go index e4b022216..4c79df383 100644 --- a/tasks/indexing/task_indexing.go +++ b/tasks/indexing/task_indexing.go @@ -257,7 +257,9 @@ func (i *IndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do } } - defer reader.Close() + defer func() { + _ = reader.Close() + }() startTime := time.Now() diff --git a/tasks/indexing/task_ipni.go b/tasks/indexing/task_ipni.go index a0bf65ca2..845d29327 100644 --- a/tasks/indexing/task_ipni.go +++ b/tasks/indexing/task_ipni.go @@ -280,7 +280,9 @@ func (I *IPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done b } } - defer reader.Close() + defer func() { + _ = reader.Close() + }() var isMK20 bool diff --git a/tasks/indexing/task_pdp_indexing.go b/tasks/indexing/task_pdp_indexing.go index d8607df47..bb909bae0 100644 --- a/tasks/indexing/task_pdp_indexing.go +++ b/tasks/indexing/task_pdp_indexing.go @@ -136,7 +136,9 @@ func (P *PDPIndexingTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) return false, xerrors.Errorf("getting piece reader: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() startTime := time.Now() diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go index 33030049e..ca30b8978 100644 --- a/tasks/indexing/task_pdp_ipni.go +++ b/tasks/indexing/task_pdp_ipni.go @@ -249,7 +249,9 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don return false, xerrors.Errorf("getting piece reader from piece park: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() recs := make(chan indexstore.Record, 1) @@ -537,7 +539,7 @@ func (P *PDPIPNITask) schedule(ctx context.Context, taskFunc harmonytask.AddTask // Mark deal is complete if: // 1. We don't need to announce anything // 2. Both type of announcements are done - if !(p.Announce && p.AnnouncePayload) || (p.Announced && p.AnnouncedPayload) { + if !(p.Announce && p.AnnouncePayload) || (p.Announced && p.AnnouncedPayload) { //nolint:staticcheck complete = &p.ID return false, nil } diff --git a/tasks/metadata/task_sector_expirations.go b/tasks/metadata/task_sector_expirations.go index 366e44090..89b23c23e 100644 --- a/tasks/metadata/task_sector_expirations.go +++ b/tasks/metadata/task_sector_expirations.go @@ -94,7 +94,9 @@ func (s *SectorMetadata) Do(taskID harmonytask.TaskID, stillOwned func() bool) ( } br := tx.SendBatch(ctx, batch) - defer br.Close() + defer func() { + _ = br.Close() + }() for i := 0; i < batch.Len(); i++ { _, err := br.Exec() diff --git a/tasks/pdp/data_set_create_watch.go b/tasks/pdp/data_set_create_watch.go index 1a3f2b0c2..9b13e7eb0 100644 --- a/tasks/pdp/data_set_create_watch.go +++ b/tasks/pdp/data_set_create_watch.go @@ -207,10 +207,10 @@ func extractDataSetIdFromReceipt(receipt *types.Receipt) (uint64, error) { } func getProvingPeriodChallengeWindow(ctx context.Context, ethClient *ethclient.Client, listenerAddr common.Address) (uint64, uint64, error) { - // ProvingPeriod - schedule, err := contract.NewIPDPProvingSchedule(listenerAddr, ethClient) + // Get the proving schedule from the listener (handles view contract indirection) + schedule, err := contract.GetProvingScheduleFromListener(listenerAddr, ethClient) if err != nil { - return 0, 0, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) + return 0, 0, xerrors.Errorf("failed to get proving schedule from listener: %w", err) } config, err := schedule.GetPDPConfig(&bind.CallOpts{Context: ctx}) diff --git a/tasks/pdp/notify_task.go b/tasks/pdp/notify_task.go index 0dcf237bc..94f0be92a 100644 --- a/tasks/pdp/notify_task.go +++ b/tasks/pdp/notify_task.go @@ -64,7 +64,9 @@ func (t *PDPNotifyTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d if err != nil { log.Errorw("HTTP POST request to notify_url failed", "notify_url", upload.NotifyURL, "upload_id", upload.ID, "error", err) } else { - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() // Not reading the body as per requirement log.Infow("HTTP GET request to notify_url succeeded", "notify_url", upload.NotifyURL, "upload_id", upload.ID) } diff --git a/tasks/pdp/task_aggregation.go b/tasks/pdp/task_aggregation.go index d62d8d480..87c041f4b 100644 --- a/tasks/pdp/task_aggregation.go +++ b/tasks/pdp/task_aggregation.go @@ -202,7 +202,9 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b return false, fmt.Errorf("failed to get piece reader: %w", err) } } - defer pr.Close() + defer func() { + _ = pr.Close() + }() pieceParked = true parkedPieceID = pid } else { @@ -288,8 +290,8 @@ func (a *AggregatePDPDealTask) Do(taskID harmonytask.TaskID, stillOwned func() b n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, - downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10)`, + downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10, TRUE)`, id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { @@ -348,6 +350,7 @@ func (a *AggregatePDPDealTask) schedule(ctx context.Context, taskFunc harmonytas FROM pdp_pipeline GROUP BY id HAVING bool_and(downloaded) + AND bool_and(after_commp) AND bool_and(NOT aggregated) AND bool_and(agg_task_id IS NULL);`) if err != nil { @@ -365,6 +368,7 @@ func (a *AggregatePDPDealTask) schedule(ctx context.Context, taskFunc harmonytas n, err := tx.Exec(`UPDATE pdp_pipeline SET agg_task_id = $1 WHERE id = $2 AND downloaded = TRUE + AND after_commp = TRUE AND aggregated = FALSE AND agg_task_id IS NULL`, id, deal.ID) if err != nil { diff --git a/tasks/pdp/task_commp.go b/tasks/pdp/task_commp.go new file mode 100644 index 000000000..921cef462 --- /dev/null +++ b/tasks/pdp/task_commp.go @@ -0,0 +1,326 @@ +package pdp + +import ( + "context" + "errors" + "io" + "net/url" + "strconv" + "time" + + "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/market/mk20" + commp "github.com/filecoin-project/go-fil-commp-hashhash" + "github.com/ipfs/go-cid" + "github.com/yugabyte/pgx/v5" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-commp-utils/writer" + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/storiface" +) + +type PDPCommpTask struct { + db *harmonydb.DB + sc *ffi.SealCalls + max int +} + +func NewPDPCommpTask(db *harmonydb.DB, sc *ffi.SealCalls, max int) *PDPCommpTask { + return &PDPCommpTask{ + db: db, + sc: sc, + max: max, + } +} + +func (c *PDPCommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var pieces []struct { + Pcid string `db:"piece_cid_v2"` + Ref int64 `db:"piece_ref"` + ID string `db:"id"` + } + + err = c.db.Select(ctx, &pieces, `SELECT + id, + piece_cid_v2 + FROM + pdp_pipeline + WHERE + commp_task_id = $1 + AND downloaded = TRUE`, taskID) + if err != nil { + return false, xerrors.Errorf("getting piece details: %w", err) + } + if len(pieces) != 1 { + return false, xerrors.Errorf("expected 1 piece, got %d", len(pieces)) + } + piece := pieces[0] + + pcid, err := cid.Parse(piece.Pcid) + if err != nil { + return false, xerrors.Errorf("parsing piece: %w", err) + } + + pi, err := mk20.GetPieceInfo(pcid) + if err != nil { + return false, xerrors.Errorf("getting piece info: %w", err) + } + + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = c.db.Select(ctx, &pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, piece.Ref) + if err != nil { + return false, xerrors.Errorf("getting pieceID: %w", err) + } + + if len(pieceID) != 1 { + return false, xerrors.Errorf("expected 1 pieceID, got %d", len(pieceID)) + } + + pr, err := c.sc.PieceReader(ctx, pieceID[0].PieceID) + if err != nil { + return false, xerrors.Errorf("getting piece reader: %w", err) + } + + pReader, pSz := padreader.New(pr, pi.RawSize) + + defer func() { + _ = pr.Close() + }() + + wr := new(commp.Calc) + written, err := io.CopyBuffer(wr, pReader, make([]byte, writer.CommPBuf)) + if err != nil { + return false, xerrors.Errorf("copy into commp writer: %w", err) + } + + if written != int64(pSz) { + return false, xerrors.Errorf("number of bytes written to CommP writer %d not equal to the file size %d", written, pSz) + } + + digest, size, err := wr.Digest() + if err != nil { + return false, xerrors.Errorf("computing commP failed: %w", err) + } + + calculatedCommp, err := commcid.DataCommitmentV1ToCID(digest) + if err != nil { + return false, xerrors.Errorf("computing commP failed: %w", err) + } + + if !calculatedCommp.Equals(pi.PieceCIDV1) { + return false, xerrors.Errorf("commp mismatch: calculated %s and expected %s", calculatedCommp, pi.PieceCIDV1) + } + + if pi.Size != abi.PaddedPieceSize(size) { + return false, xerrors.Errorf("pieceSize mismatch: expected %d, got %d", pi.Size, abi.PaddedPieceSize(size)) + } + + n, err := c.db.Exec(ctx, `UPDATE pdp_pipeline SET after_commp = TRUE, commp_task_id = NULL + WHERE id = $1 + AND piece_cid_v2 = $2 + AND downloaded = TRUE + AND after_commp = FALSE + AND commp_task_id = $3`, + piece.ID, piece.Pcid, taskID) + + if err != nil { + return false, xerrors.Errorf("store commp success: updating pdp pipeline: %w", err) + } + + if n != 1 { + return false, xerrors.Errorf("store commp success: updated %d rows", n) + } + + return true, nil + +} + +func (c *PDPCommpTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) { + // CommP task can be of 2 types + // 1. Using ParkPiece pieceRef + // 2. Using remote HTTP reader + // ParkPiece should be scheduled on same node which has the piece + // Remote HTTP ones can be scheduled on any node + + ctx := context.Background() + + var tasks []struct { + TaskID harmonytask.TaskID `db:"commp_task_id"` + StorageID string `db:"storage_id"` + Url *string `db:"url"` + } + + indIDs := make([]int64, len(ids)) + for i, id := range ids { + indIDs[i] = int64(id) + } + + comm, err := c.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + err = tx.Select(&tasks, ` SELECT + commp_task_id, + url + FROM + market_mk12_deal_pipeline + WHERE + commp_task_id = ANY ($1) + + UNION ALL + + SELECT + commp_task_id, + url + FROM + market_mk20_pipeline + WHERE + commp_task_id = ANY ($1); + `, indIDs) + if err != nil { + return false, xerrors.Errorf("failed to get deal details from DB: %w", err) + } + + if storiface.FTPiece != 32 { + panic("storiface.FTPiece != 32") + } + + for _, task := range tasks { + if task.Url != nil { + goUrl, err := url.Parse(*task.Url) + if err != nil { + return false, xerrors.Errorf("parsing data URL: %w", err) + } + if goUrl.Scheme == "pieceref" { + refNum, err := strconv.ParseInt(goUrl.Opaque, 10, 64) + if err != nil { + return false, xerrors.Errorf("parsing piece reference number: %w", err) + } + + // get pieceID + var pieceID []struct { + PieceID storiface.PieceNumber `db:"piece_id"` + } + err = tx.Select(&pieceID, `SELECT piece_id FROM parked_piece_refs WHERE ref_id = $1`, refNum) + if err != nil { + return false, xerrors.Errorf("getting pieceID: %w", err) + } + + var sLocation string + + err = tx.QueryRow(` + SELECT storage_id FROM sector_location + WHERE miner_id = 0 AND sector_num = $1 AND sector_filetype = 32`, pieceID[0].PieceID).Scan(&sLocation) + + if err != nil { + return false, xerrors.Errorf("failed to get storage location from DB: %w", err) + } + + task.StorageID = sLocation + } + } + } + + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return nil, err + } + + if !comm { + return nil, xerrors.Errorf("failed to commit the transaction") + } + + ls, err := c.sc.LocalStorage(ctx) + if err != nil { + return nil, xerrors.Errorf("getting local storage: %w", err) + } + + acceptables := map[harmonytask.TaskID]bool{} + + for _, t := range ids { + acceptables[t] = true + } + + for _, t := range tasks { + if _, ok := acceptables[t.TaskID]; !ok { + continue + } + + for _, l := range ls { + if string(l.ID) == t.StorageID { + return &t.TaskID, nil + } + } + } + + // If no local pieceRef was found then just return first TaskID + return &ids[0], nil +} + +func (c *PDPCommpTask) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(c.max), + Name: "PDPCommP", + Cost: resources.Resources{ + Cpu: 1, + Ram: 1 << 30, + }, + MaxFailures: 3, + IAmBored: passcall.Every(3*time.Second, func(taskFunc harmonytask.AddTaskFunc) error { + return c.schedule(context.Background(), taskFunc) + }), + } +} + +func (c *PDPCommpTask) schedule(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + var stop bool + for !stop { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + stop = true // assume we're done until we find a task to schedule + + var did string + err := tx.QueryRow(`SELECT id FROM pdp_pipeline + WHERE commp_task_id IS NULL + AND after_commp = FALSE + AND downloaded = TRUE`).Scan(&did) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to query pdp_pipeline: %w", err) + } + if did == "" { + return false, xerrors.Errorf("no valid deal ID found for scheduling") + } + + _, err = tx.Exec(`UPDATE pdp_pipeline SET commp_task_id = $1 WHERE id = $2 AND commp_task_id IS NULL AND after_commp = FALSE AND downloaded = TRUE`, id, did) + if err != nil { + return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) + } + + stop = false // we found a task to schedule, keep going + return true, nil + }) + + } + + return nil +} + +func (c *PDPCommpTask) Adder(taskFunc harmonytask.AddTaskFunc) {} + +var _ = harmonytask.Reg(&PDPCommpTask{}) +var _ harmonytask.TaskInterface = &PDPCommpTask{} diff --git a/tasks/pdp/task_init_pp.go b/tasks/pdp/task_init_pp.go index 1ab30d594..c067f162e 100644 --- a/tasks/pdp/task_init_pp.go +++ b/tasks/pdp/task_init_pp.go @@ -131,10 +131,10 @@ func (ipp *InitProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func( return false, xerrors.Errorf("failed to get listener address for data set %d: %w", dataSetID, err) } - // Determine the next challenge window start by consulting the listener - provingSchedule, err := contract.NewIPDPProvingSchedule(listenerAddr, ipp.ethClient) + // Get the proving schedule from the listener (handles view contract indirection) + provingSchedule, err := contract.GetProvingScheduleFromListener(listenerAddr, ipp.ethClient) if err != nil { - return false, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) + return false, xerrors.Errorf("failed to get proving schedule from listener: %w", err) } config, err := provingSchedule.GetPDPConfig(&bind.CallOpts{Context: ctx}) diff --git a/tasks/pdp/task_next_pp.go b/tasks/pdp/task_next_pp.go index c1458e970..30c2d1954 100644 --- a/tasks/pdp/task_next_pp.go +++ b/tasks/pdp/task_next_pp.go @@ -118,10 +118,10 @@ func (n *NextProvingPeriodTask) Do(taskID harmonytask.TaskID, stillOwned func() return false, xerrors.Errorf("failed to get listener address for data set %d: %w", dataSetID, err) } - // Determine the next challenge window start by consulting the listener - provingSchedule, err := contract.NewIPDPProvingSchedule(listenerAddr, n.ethClient) + // Get the proving schedule from the listener (handles view contract indirection) + provingSchedule, err := contract.GetProvingScheduleFromListener(listenerAddr, n.ethClient) if err != nil { - return false, xerrors.Errorf("failed to create proving schedule binding, check that listener has proving schedule methods: %w", err) + return false, xerrors.Errorf("failed to get proving schedule from listener: %w", err) } next_prove_at, err := provingSchedule.NextPDPChallengeWindowStart(nil, big.NewInt(dataSetID)) diff --git a/tasks/pdp/task_prove.go b/tasks/pdp/task_prove.go index a1f2f3fa9..8f3c4dd16 100644 --- a/tasks/pdp/task_prove.go +++ b/tasks/pdp/task_prove.go @@ -439,7 +439,9 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 if err != nil { return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get piece reader: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() // Build Merkle tree from padded input memTree, err := proof.BuildSha254Memtree(reader, pi.Size.Unpadded()) @@ -498,7 +500,9 @@ func (p *ProveTask) proveRoot(ctx context.Context, dataSetID int64, rootId int64 if err != nil { return contract.IPDPTypesProof{}, xerrors.Errorf("failed to get reader: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() fileRemaining := int64(reportedSize) - offset diff --git a/tasks/pdp/task_save_cache.go b/tasks/pdp/task_save_cache.go index e4fbf7d2c..df33c609d 100644 --- a/tasks/pdp/task_save_cache.go +++ b/tasks/pdp/task_save_cache.go @@ -93,7 +93,9 @@ func (t *TaskPDPSaveCache) Do(taskID harmonytask.TaskID, stillOwned func() bool) if err != nil { return false, xerrors.Errorf("failed to get shared piece reader: %w", err) } - defer reader.Close() + defer func() { + _ = reader.Close() + }() n, err := io.CopyBuffer(cp, reader, make([]byte, 4<<20)) if err != nil { @@ -182,7 +184,7 @@ func (t *TaskPDPSaveCache) schedule(ctx context.Context, taskFunc harmonytask.Ad return false, xerrors.Errorf("no valid deal ID found for scheduling") } - _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1 WHERE id = $2 AND after_save_cache = FALSE AND after_add_piece_msg = TRUE`, id, did) + _, err = tx.Exec(`UPDATE pdp_pipeline SET save_cache_task_id = $1 WHERE id = $2 AND after_save_cache = FALSE AND after_add_piece_msg = TRUE AND save_cache_task_id IS NULL`, id, did) if err != nil { return false, xerrors.Errorf("failed to update pdp_pipeline: %w", err) } @@ -568,7 +570,7 @@ func (cp *Calc) hashSlab254(layerIdx uint, collectSnapshot bool, slab []byte) { func NewCommPWithSize(size uint64) *Calc { c := new(Calc) - c.state.size = size + c.size = size c.snapshotLayerIndex(size, false) @@ -645,7 +647,7 @@ func (cp *Calc) DigestWithSnapShot() ([]byte, uint64, int, []NodeDigest, error) func NewCommPWithSizeForTest(size uint64) *Calc { c := new(Calc) - c.state.size = size + c.size = size c.snapshotLayerIndex(size, true) diff --git a/tasks/piece/task_aggregate_chunks.go b/tasks/piece/task_aggregate_chunks.go index cc8d66224..a42b9fbd0 100644 --- a/tasks/piece/task_aggregate_chunks.go +++ b/tasks/piece/task_aggregate_chunks.go @@ -143,7 +143,9 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo return false, fmt.Errorf("failed to get piece reader: %w", err) } } - defer pr.Close() + defer func() { + _ = pr.Close() + }() pieceParked = true parkedPieceID = pid } else { @@ -314,8 +316,8 @@ func (a *AggregateChunksTask) Do(taskID harmonytask.TaskID, stillOwned func() bo n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, - downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10)`, + downloaded, deal_aggregation, aggr_index, indexing, announce, announce_payload, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, $9, $10, TRUE)`, id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { diff --git a/tasks/sealsupra/supra_config.go b/tasks/sealsupra/supra_config.go index bad4cd961..c4ba23e4a 100644 --- a/tasks/sealsupra/supra_config.go +++ b/tasks/sealsupra/supra_config.go @@ -89,7 +89,7 @@ func GetSystemInfo() (*SystemInfo, error) { packageRegex := regexp.MustCompile(`Package L#(\d+)`) var currentL3Cores int - var lastL3Index int = -1 + var lastL3Index = -1 var threadCount int for scanner.Scan() { diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 7f03d653b..97577d744 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -83,6 +83,7 @@ func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { } }() d.processMK20DealPieces(ctx) + d.downloadMk20Deal(ctx) d.processMK20DealAggregation(ctx) d.processMK20DealIngestion(ctx) } @@ -291,8 +292,8 @@ func (d *CurioStorageDealMarket) insertDealInPipelineForUpload(ctx context.Conte n, err := tx.Exec(`INSERT INTO pdp_pipeline ( id, client, piece_cid_v2, data_set_id, extra_data, piece_ref, - downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload) - VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10)`, + downloaded, deal_aggregation, aggr_index, aggregated, indexing, announce, announce_payload, after_commp) + VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, TRUE, $8, $9, $10, TRUE)`, id, deal.Client, deal.Data.PieceCID.String(), *pdp.DataSetID, pdp.ExtraData, pieceRefID, deal.Data.Format.Aggregate.Type, retv.Indexing, retv.AnnouncePiece, retv.AnnouncePayload) if err != nil { @@ -385,8 +386,8 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 refIds = append(refIds, refID) } - n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) VALUES ($1, $2, $3, $4, $5)`, - dealID, pi.PieceCIDV1.String(), pi.Size, mk20.ProductNameDDOV1, refIds) + n, err := tx.Exec(`INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) VALUES ($1, $2, $3, $4)`, + dealID, deal.Data.PieceCID.String(), mk20.ProductNameDDOV1, refIds) if err != nil { return xerrors.Errorf("inserting mk20 download pipeline: %w", err) } @@ -435,10 +436,11 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 // Find all unique pieces where data source is HTTP type downloadkey struct { - ID string - PieceCID cid.Cid - Size abi.PaddedPieceSize - RawSize uint64 + ID string + PieceCIDV2 cid.Cid + PieceCID cid.Cid + Size abi.PaddedPieceSize + RawSize uint64 } toDownload := make(map[downloadkey][]mk20.HttpUrl) @@ -448,11 +450,11 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 return xerrors.Errorf("getting piece info: %w", err) } if piece.SourceHTTP != nil { - urls, ok := toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] + urls, ok := toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] if ok { - toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) + toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size}] = append(urls, piece.SourceHTTP.URLs...) } else { - toDownload[downloadkey{ID: dealID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs + toDownload[downloadkey{ID: dealID, PieceCIDV2: piece.PieceCID, PieceCID: spi.PieceCIDV1, Size: spi.Size, RawSize: spi.RawSize}] = piece.SourceHTTP.URLs } } } @@ -484,15 +486,15 @@ func insertPiecesInTransaction(ctx context.Context, tx *harmonydb.Tx, deal *mk20 SELECT id, $4, $5, FALSE FROM selected_piece RETURNING ref_id ) - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) - VALUES ($6, $1, $2, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) - ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE + INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) + VALUES ($6, $8, $7, ARRAY[(SELECT ref_id FROM inserted_ref)]) + ON CONFLICT (id, piece_cid_v2, product) DO UPDATE SET ref_ids = array_append( market_mk20_download_pipeline.ref_ids, (SELECT ref_id FROM inserted_ref) ) WHERE NOT market_mk20_download_pipeline.ref_ids @> ARRAY[(SELECT ref_id FROM inserted_ref)];`, - k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, mk20.ProductNameDDOV1) + k.PieceCID.String(), k.Size, k.RawSize, src.URL, headers, k.ID, mk20.ProductNameDDOV1, k.PieceCIDV2.String()) } if batch.Len() > batchSize { @@ -601,12 +603,12 @@ func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { } func (d *CurioStorageDealMarket) processMk20Pieces(ctx context.Context, piece MK20PipelinePiece) error { - err := d.downloadMk20Deal(ctx, piece) - if err != nil { - return err - } + //err := d.downloadMk20Deal(ctx, piece) + //if err != nil { + // return err + //} - err = d.findOfflineURLMk20Deal(ctx, piece) + err := d.findOfflineURLMk20Deal(ctx, piece) if err != nil { return err } @@ -625,68 +627,74 @@ func (d *CurioStorageDealMarket) processMk20Pieces(ctx context.Context, piece MK } // downloadMk20Deal handles the downloading process of an MK20 pipeline piece by scheduling it in the database and updating its status. -// If the pieces are part of an aggregation deal then we download for short term otherwise we check if piece needs to be indexed. -// If indexing is true then we download for long term to avoid the need to have unsealed copy -func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK20PipelinePiece) error { - if !piece.Downloaded && piece.Started { - _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - var refid int64 - err = tx.QueryRow(`SELECT u.ref_id FROM ( - SELECT unnest(dp.ref_ids) AS ref_id - FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - ) u - JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id - JOIN parked_pieces pp ON pp.id = pr.piece_id - WHERE pp.complete = TRUE - LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&refid) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return false, nil - } - return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) - } - - // Remove other ref_ids from piece_park_refs - _, err = tx.Exec(`DELETE FROM parked_piece_refs - WHERE ref_id IN ( - SELECT unnest(dp.ref_ids) - FROM market_mk20_download_pipeline dp - WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - ) - AND ref_id != $5;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1, refid) - if err != nil { - return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) - } - - _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, - piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1) - if err != nil { - return false, xerrors.Errorf("failed to delete piece from download table: %w", err) - } - - pieceIDUrl := url.URL{ - Scheme: "pieceref", - Opaque: fmt.Sprintf("%d", refid), - } - - _, err = tx.Exec(`UPDATE market_mk20_pipeline SET downloaded = TRUE, url = $1 - WHERE id = $2 - AND piece_cid = $3 - AND piece_size = $4`, - pieceIDUrl.String(), piece.ID, piece.PieceCID, piece.PieceSize) - if err != nil { - return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) - } - piece.Downloaded = true - return true, nil - }, harmonydb.OptionRetry()) +// If the pieces are part of an aggregation deal then we download for short term otherwise, +// we download for long term to avoid the need to have unsealed copy +func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context) { + n, err := d.db.Exec(ctx, `SELECT mk20_ddo_mark_downloaded($1)`, mk20.ProductNameDDOV1) + if err != nil { + log.Errorf("failed to mark PDP downloaded piece: %v", err) - if err != nil { - return xerrors.Errorf("failed to schedule the deal for download: %w", err) - } } - return nil + log.Debugf("Succesfully marked %d PDP pieces as downloaded", n) + + //if !piece.Downloaded && piece.Started { + //_, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + // var refid int64 + // err = tx.QueryRow(`SELECT u.ref_id FROM ( + // SELECT unnest(dp.ref_ids) AS ref_id + // FROM market_mk20_download_pipeline dp + // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + // ) u + // JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id + // JOIN parked_pieces pp ON pp.id = pr.piece_id + // WHERE pp.complete = TRUE + // LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&refid) + // if err != nil { + // if errors.Is(err, pgx.ErrNoRows) { + // return false, nil + // } + // return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) + // } + // + // // Remove other ref_ids from piece_park_refs + // _, err = tx.Exec(`DELETE FROM parked_piece_refs + // WHERE ref_id IN ( + // SELECT unnest(dp.ref_ids) + // FROM market_mk20_download_pipeline dp + // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + // ) + // AND ref_id != $5;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1, refid) + // if err != nil { + // return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) + // } + // + // _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, + // piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1) + // if err != nil { + // return false, xerrors.Errorf("failed to delete piece from download table: %w", err) + // } + // + // pieceIDUrl := url.URL{ + // Scheme: "pieceref", + // Opaque: fmt.Sprintf("%d", refid), + // } + // + // _, err = tx.Exec(`UPDATE market_mk20_pipeline SET downloaded = TRUE, url = $1 + // WHERE id = $2 + // AND piece_cid = $3 + // AND piece_size = $4`, + // pieceIDUrl.String(), piece.ID, piece.PieceCID, piece.PieceSize) + // if err != nil { + // return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) + // } + // piece.Downloaded = true + // return true, nil + //}, harmonydb.OptionRetry()) + // + //if err != nil { + // return xerrors.Errorf("failed to schedule the deal for download: %w", err) + //} + //} } // findOfflineURLMk20Deal find the URL for offline piece. In MK20, we don't work directly with remote pieces, we download them @@ -790,10 +798,10 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie RETURNING ref_id ), upsert_pipeline AS ( - INSERT INTO market_mk20_download_pipeline (id, piece_cid, piece_size, product, ref_ids) - SELECT $1, $2, $3, $7, array_agg(ref_id) + INSERT INTO market_mk20_download_pipeline (id, piece_cid_v2, product, ref_ids) + SELECT $1, $8, $7, array_agg(ref_id) FROM inserted_ref - ON CONFLICT (id, piece_cid, piece_size, product) DO UPDATE + ON CONFLICT (id, piece_cid_v2, product) DO UPDATE SET ref_ids = ( SELECT array( SELECT DISTINCT r @@ -804,7 +812,7 @@ func (d *CurioStorageDealMarket) findOfflineURLMk20Deal(ctx context.Context, pie UPDATE market_mk20_pipeline SET started = TRUE WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND started = FALSE;`, - piece.ID, piece.PieceCID, piece.PieceSize, rawSize, urlString, hdrs, mk20.ProductNameDDOV1) + piece.ID, piece.PieceCID, piece.PieceSize, rawSize, urlString, hdrs, mk20.ProductNameDDOV1, piece.PieceCIDV2) if err != nil { return false, xerrors.Errorf("failed to start download for offline deal using PieceLocator: %w", err) } diff --git a/tasks/storage-market/storage_market.go b/tasks/storage-market/storage_market.go index 7b1251c08..831d6c194 100644 --- a/tasks/storage-market/storage_market.go +++ b/tasks/storage-market/storage_market.go @@ -139,10 +139,6 @@ func NewCurioStorageDealMarket(miners []address.Address, db *harmonydb.DB, cfg * func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { var err error - if len(d.miners) == 0 { - // Do not start the poller if no minerID present - return nil - } d.MK12Handler, err = mk12.NewMK12Handler(d.miners, d.db, d.si, d.api, d.cfg, d.as) if err != nil { return err @@ -176,15 +172,17 @@ func (d *CurioStorageDealMarket) StartMarket(ctx context.Context) error { return err } - if d.cfg.Ingest.DoSnap { - d.pin, err = storageingest.NewPieceIngesterSnap(ctx, d.db, d.api, d.miners, d.cfg) - } else { - d.pin, err = storageingest.NewPieceIngester(ctx, d.db, d.api, d.miners, d.cfg) + if len(d.miners) > 0 { + if d.cfg.Ingest.DoSnap { + d.pin, err = storageingest.NewPieceIngesterSnap(ctx, d.db, d.api, d.miners, d.cfg) + } else { + d.pin, err = storageingest.NewPieceIngester(ctx, d.db, d.api, d.miners, d.cfg) + } + if err != nil { + return err + } } - if err != nil { - return err - } go d.runPoller(ctx) return nil diff --git a/tasks/storage-market/task_aggregation.go b/tasks/storage-market/task_aggregation.go index d146c05ce..ec86eadf0 100644 --- a/tasks/storage-market/task_aggregation.go +++ b/tasks/storage-market/task_aggregation.go @@ -222,7 +222,9 @@ func (a *AggregateDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool return false, fmt.Errorf("failed to get piece reader: %w", err) } } - defer pr.Close() + defer func() { + _ = pr.Close() + }() pieceParked = true parkedPieceID = pid } else { diff --git a/tasks/winning/winning_task.go b/tasks/winning/winning_task.go index 6fef12eff..504fa648d 100644 --- a/tasks/winning/winning_task.go +++ b/tasks/winning/winning_task.go @@ -19,7 +19,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/proof" - prooftypes "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps" @@ -256,7 +255,7 @@ func (t *WinPostTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don log.Infow("WinPostTask won election", "tipset", types.LogCids(base.TipSet.Cids()), "miner", maddr, "round", round, "eproof", eproof) // winning PoSt - var wpostProof []prooftypes.PoStProof + var wpostProof []proof.PoStProof { buf := new(bytes.Buffer) if err := maddr.MarshalCBOR(buf); err != nil { diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index 4b7e4770e..d635171b1 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -833,8 +833,8 @@ type MK12DealPipeline struct { CreatedAt time.Time `db:"created_at" json:"created_at"` } -// MK20DealPipeline represents a record from market_mk12_deal_pipeline table -type MK20DealPipeline struct { +// MK20DealPipeline represents a record from market_mk20_ddo_pipeline table +type MK20DDOPipeline struct { ID string `db:"id" json:"id"` SpId int64 `db:"sp_id" json:"sp_id"` Contract string `db:"contract" json:"contract"` @@ -883,8 +883,9 @@ type PieceInfoMK12Deals struct { } type PieceInfoMK20Deals struct { - Deal *MK20StorageDeal `json:"deal"` - Pipeline *MK20DealPipeline `json:"mk20_pipeline,omitempty"` + Deal *MK20StorageDeal `json:"deal"` + DDOPipeline *MK20DDOPipeline `json:"mk20_ddo_pipeline,omitempty"` + PDPPipeline *MK20PDPPipeline `json:"mk20_pdp_pipeline,omitempty"` } // PieceDealDetailEntry combines a deal and its pipeline @@ -1061,12 +1062,12 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe } mk20deals[i] = &MK20StorageDeal{ - Deal: deal, - Error: Err, + Deal: deal, + DDOErr: Err, } } - var mk20Pipelines []MK20DealPipeline + var mk20Pipelines []MK20DDOPipeline err = a.deps.DB.Select(ctx, &mk20Pipelines, ` SELECT created_at, @@ -1103,15 +1104,53 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe FROM market_mk20_pipeline WHERE id = ANY($1)`, ids) if err != nil { - return nil, xerrors.Errorf("failed to query mk20 pipelines: %w", err) + return nil, xerrors.Errorf("failed to query mk20 DDO pipelines: %w", err) } - mk20pipelineMap := make(map[string]MK20DealPipeline) + var mk20PDPPipelines []MK20PDPPipeline + err = a.deps.DB.Select(ctx, &mk20PDPPipelines, ` + SELECT + created_at, + id, + client, + piece_cid_v2, + indexing, + announce, + announce_payload, + downloaded, + commp_task_id, + after_commp, + deal_aggregation, + aggr_index, + agg_task_id, + aggregated, + add_piece_task_id, + after_add_piece, + after_add_piece_msg, + save_cache_task_id, + after_save_cache, + indexing_created_at, + indexing_task_id, + indexed, + complete + FROM pdp_pipeline + WHERE id = ANY($1)`, ids) + if err != nil { + return nil, xerrors.Errorf("failed to query mk20 PDP pipelines: %w", err) + } + + mk20pipelineMap := make(map[string]MK20DDOPipeline) for _, pipeline := range mk20Pipelines { pipeline := pipeline mk20pipelineMap[pipeline.ID] = pipeline } + mk20PDPpipelineMap := make(map[string]MK20PDPPipeline) + for _, pipeline := range mk20PDPPipelines { + pipeline := pipeline + mk20PDPpipelineMap[pipeline.ID] = pipeline + } + ret := &PieceDealDetailEntry{} for _, deal := range mk12Deals { @@ -1131,9 +1170,14 @@ func (a *WebRPC) PieceDealDetail(ctx context.Context, pieceCid string) (*PieceDe Deal: deal, } if pipeline, exists := mk20pipelineMap[deal.Deal.Identifier.String()]; exists { - entry.Pipeline = &pipeline + entry.DDOPipeline = &pipeline } else { - entry.Pipeline = nil // Pipeline may not exist for processed and active deals + entry.DDOPipeline = nil // Pipeline may not exist for processed and active deals + } + if pipeline, exists := mk20PDPpipelineMap[deal.Deal.Identifier.String()]; exists { + entry.PDPPipeline = &pipeline + } else { + entry.PDPPipeline = nil } if ret.MK20 == nil { ret.MK20 = make([]PieceInfoMK20Deals, 0) diff --git a/web/api/webrpc/market_20.go b/web/api/webrpc/market_2.go similarity index 50% rename from web/api/webrpc/market_20.go rename to web/api/webrpc/market_2.go index 533d2aca3..51020a6ce 100644 --- a/web/api/webrpc/market_20.go +++ b/web/api/webrpc/market_2.go @@ -23,8 +23,10 @@ import ( ) type MK20StorageDeal struct { - Deal *mk20.Deal `json:"deal"` - Error sql.NullString `json:"error"` + Deal *mk20.Deal `json:"deal"` + DDOErr sql.NullString `json:"ddoerr"` + PDPErr sql.NullString `json:"pdperr"` + DDOId sql.NullInt64 `json:"ddoid"` } func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20StorageDeal, error) { @@ -60,7 +62,20 @@ func (a *WebRPC) MK20DDOStorageDeal(ctx context.Context, id string) (*MK20Storag return nil, fmt.Errorf("unmarshal ddov1: %w", err) } if dddov1.Error != "" { - ret.Error = sql.NullString{String: dddov1.Error, Valid: true} + ret.DDOErr = sql.NullString{String: dddov1.Error, Valid: true} + } + if dddov1.DealID > 0 { + ret.DDOId = sql.NullInt64{Int64: dddov1.DealID, Valid: true} + } + } + + if len(dbDeal.PDPV1) > 0 && string(dbDeal.PDPV1) != "null" { + var pdpv1 mk20.DBPDPV1 + if err := json.Unmarshal(dbDeal.PDPV1, &pdpv1); err != nil { + return nil, fmt.Errorf("unmarshal pdpv1: %w", err) + } + if pdpv1.Error != "" { + ret.PDPErr = sql.NullString{String: pdpv1.Error, Valid: true} } } @@ -107,7 +122,7 @@ func (a *WebRPC) MK20DDOStorageDeals(ctx context.Context, limit int, offset int) return mk20Summaries, nil } -func (a *WebRPC) MK20DealPipelines(ctx context.Context, limit int, offset int) ([]*MK20DealPipeline, error) { +func (a *WebRPC) MK20DDOPipelines(ctx context.Context, limit int, offset int) ([]*MK20DDOPipeline, error) { if limit <= 0 { limit = 25 } @@ -118,7 +133,7 @@ func (a *WebRPC) MK20DealPipelines(ctx context.Context, limit int, offset int) ( offset = 0 } - var pipelines []*MK20DealPipeline + var pipelines []*MK20DDOPipeline err := a.deps.DB.Select(ctx, &pipelines, ` SELECT created_at, @@ -817,3 +832,697 @@ func (a *WebRPC) ChunkUploadStatus(ctx context.Context, idStr string) (*UploadSt Status: status, }, nil } + +// MK20PDPPipeline represents a record from market_mk20_PDP_pipeline table +type MK20PDPPipeline struct { + ID string `db:"id" json:"id"` + Client string `db:"client" json:"client"` + PieceCidV2 string `db:"piece_cid_v2" json:"piece_cid_v2"` + Indexing bool `db:"indexing" json:"indexing"` + Announce bool `db:"announce" json:"announce"` + AnnouncePayload bool `db:"announce_payload" json:"announce_payload"` + + Downloaded bool `db:"downloaded" json:"downloaded"` + + CommpTaskId sql.NullInt64 `db:"commp_task_id" json:"commp_task_id"` + AfterCommp bool `db:"after_commp" json:"after_commp"` + + DealAggregation int `db:"deal_aggregation" json:"deal_aggregation"` + AggregationIndex int64 `db:"aggr_index" json:"aggr_index"` + AggregationTaskID sql.NullInt64 `db:"agg_task_id" json:"agg_task_id"` + Aggregated bool `db:"aggregated" json:"aggregated"` + + AddPieceTaskID sql.NullInt64 `db:"add_piece_task_id" json:"add_piece_task_id"` + AfterAddPiece bool `db:"after_add_piece" json:"after_add_piece"` + + AfterAddPieceMsg bool `db:"after_add_piece_msg" json:"after_add_piece_msg"` + + SaveCacheTaskID sql.NullInt64 `db:"save_cache_task_id" json:"save_cache_task_id"` + AfterSaveCache bool `db:"after_save_cache" json:"after_save_cache"` + + IndexingCreatedAt sql.NullTime `db:"indexing_created_at" json:"indexing_created_at"` + IndexingTaskId sql.NullInt64 `db:"indexing_task_id" json:"indexing_task_id"` + Indexed bool `db:"indexed" json:"indexed"` + + Complete bool `db:"complete" json:"complete"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + + Miner string `db:"-" json:"miner"` +} + +type MK20PDPDealList struct { + ID string `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + PieceCidV2 sql.NullString `db:"piece_cid_v2" json:"piece_cid_v2"` + Processed bool `db:"processed" json:"processed"` + Error sql.NullString `db:"error" json:"error"` +} + +func (a *WebRPC) MK20PDPStorageDeals(ctx context.Context, limit int, offset int) ([]*MK20PDPDealList, error) { + var pdpSummaries []*MK20PDPDealList + + err := a.deps.DB.Select(ctx, &pdpSummaries, `SELECT + d.created_at, + d.id, + d.piece_cid_v2, + (d.pdp_v1->>'error')::text AS error, + (d.complete->>'complete'::boolean) as processed + FROM market_mk20_deal d + WHERE d.pdp_v1 IS NOT NULL AND d.pdp_v1 != 'null' + ORDER BY d.created_at DESC + LIMIT $1 OFFSET $2;`, limit, offset) + if err != nil { + return nil, fmt.Errorf("failed to fetch PDP deal list: %w", err) + } + + return pdpSummaries, nil +} + +func (a *WebRPC) MK20PDPPipelines(ctx context.Context, limit int, offset int) ([]*MK20PDPPipeline, error) { + if limit <= 0 { + limit = 25 + } + if limit > 100 { + limit = 100 + } + if offset < 0 { + offset = 0 + } + + var pipelines []*MK20PDPPipeline + err := a.deps.DB.Select(ctx, &pipelines, ` + SELECT + created_at, + id, + client, + piece_cid_v2, + indexing, + announce, + announce_payload, + downloaded, + commp_task_id, + after_commp, + deal_aggregation, + aggr_index, + agg_task_id, + aggregated, + add_piece_task_id, + after_add_piece, + after_add_piece_msg, + save_cache_task_id, + after_save_cache, + indexing_created_at, + indexing_task_id, + indexed, + complete + FROM pdp_pipeline + ORDER BY created_at DESC + LIMIT $1 OFFSET $2`, limit, offset) + if err != nil { + return nil, fmt.Errorf("failed to fetch pdp pipelines: %w", err) + } + + return pipelines, nil +} + +type MK20PDPPipelineFailedStats struct { + DownloadingFailed int64 + CommPFailed int64 + AggFailed int64 + AddPiece int64 + SaveCache int64 + IndexFailed int64 +} + +func (a *WebRPC) MK20PDPPipelineFailedTasks(ctx context.Context) (*MK20PDPPipelineFailedStats, error) { + // We'll create a similar query, but this time we coalesce the task IDs from harmony_task. + // If the join fails (no matching harmony_task), all joined fields for that task will be NULL. + // We detect failure by checking that xxx_task_id IS NOT NULL, after_xxx = false, and that no task record was found in harmony_task. + + const query = ` + WITH pipeline_data AS ( + SELECT + dp.id, + dp.complete, + dp.commp_task_id, + dp.agg_task_id, + dp.add_piece_task_id, + dp.save_cache_task_id, + dp.indexing_task_id, + dp.after_commp, + dp.aggregated, + dp.after_add_piece, + dp.after_save_cache, + t.downloading_task_id + FROM pdp_pipeline dp + LEFT JOIN market_mk20_download_pipeline mdp + ON mdp.id = dp.id + AND mdp.piece_cid_v2 = dp.piece_cid_v2 + AND mdp.product = $1 + LEFT JOIN LATERAL ( + SELECT pp.task_id AS downloading_task_id + FROM unnest(mdp.ref_ids) AS r(ref_id) + JOIN parked_piece_refs pr ON pr.ref_id = r.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.complete = FALSE + LIMIT 1 + ) t ON TRUE + WHERE dp.complete = FALSE + ), + tasks AS ( + SELECT p.*, + dt.id AS downloading_tid, + ct.id AS commp_tid, + at.id AS agg_tid, + ap.id as add_piece_tid, + sc.id as save_cache_tid, + it.id AS index_tid + FROM pipeline_data p + LEFT JOIN harmony_task dt ON dt.id = p.downloading_task_id + LEFT JOIN harmony_task ct ON ct.id = p.commp_task_id + LEFT JOIN harmony_task pt ON at.id = p.agg_task_id + LEFT JOIN harmony_task pt ON ap.id = p.add_piece_task_id + LEFT JOIN harmony_task pt ON sc.id = p.save_cache_task_id + LEFT JOIN harmony_task it ON it.id = p.indexing_task_id + ) + SELECT + -- Downloading failed: + -- downloading_task_id IS NOT NULL, after_commp = false (haven't completed commp stage), + -- and downloading_tid IS NULL (no harmony_task record) + COUNT(*) FILTER ( + WHERE downloading_task_id IS NOT NULL + AND after_commp = false + AND downloading_tid IS NULL + ) AS downloading_failed, + + -- CommP (verify) failed: + -- commp_task_id IS NOT NULL, after_commp = false, commp_tid IS NULL + COUNT(*) FILTER ( + WHERE commp_task_id IS NOT NULL + AND after_commp = false + AND commp_tid IS NULL + ) AS commp_failed, + + -- Aggregation failed: + -- agg_task_id IS NOT NULL, aggregated = false, agg_tid IS NULL + COUNT(*) FILTER ( + WHERE agg_task_id IS NOT NULL + AND aggregated = false + AND agg_tid IS NULL + ) AS agg_failed, + + -- Add Piece failed: + -- add_piece_task_id IS NOT NULL, after_add_piece = false, add_piece_tid IS NULL + COUNT(*) FILTER ( + WHERE add_piece_task_id IS NOT NULL + AND after_add_piece = false + AND add_piece_tid IS NULL + ) AS add_piece_failed, + + -- Save Cache failed: + -- save_cache_task_id IS NOT NULL, after_save_cache = false, save_cache_tid IS NULL + COUNT(*) FILTER ( + WHERE save_cache_task_id IS NOT NULL + AND after_save_cache = false + AND save_cache_tid IS NULL + ) AS save_cache_failed, + + -- Index failed: + -- indexing_task_id IS NOT NULL and if we assume indexing is after find_deal: + -- If indexing_task_id is set, we are presumably at indexing stage. + -- If index_tid IS NULL (no task found), then it's failed. + -- We don't have after_index, now at indexing. + COUNT(*) FILTER ( + WHERE indexing_task_id IS NOT NULL + AND index_tid IS NULL + AND after_save_cache = true + ) AS index_failed + FROM tasks + ` + + var c []struct { + DownloadingFailed int64 `db:"downloading_failed"` + CommPFailed int64 `db:"commp_failed"` + AggFailed int64 `db:"agg_failed"` + AddPieceFailed int64 `db:"add_piece_failed"` + SaveCacheFailed int64 `db:"save_cache_failed"` + IndexFailed int64 `db:"index_failed"` + } + + err := a.deps.DB.Select(ctx, &c, query, mk20.ProductNamePDPV1) + if err != nil { + return nil, xerrors.Errorf("failed to run failed task query: %w", err) + } + + counts := c[0] + + return &MK20PDPPipelineFailedStats{ + DownloadingFailed: counts.DownloadingFailed, + CommPFailed: counts.CommPFailed, + AggFailed: counts.AggFailed, + AddPiece: counts.AddPieceFailed, + SaveCache: counts.SaveCacheFailed, + IndexFailed: counts.IndexFailed, + }, nil +} + +func (a *WebRPC) MK20BulkRestartFailedPDPTasks(ctx context.Context, taskType string) error { + didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + var rows *harmonydb.Query + var err error + + switch taskType { + case "downloading": + rows, err = tx.Query(` + SELECT + t.task_id + FROM pdp_pipeline dp + LEFT JOIN market_mk20_download_pipeline mdp + ON mdp.id = dp.id + AND mdp.piece_cid_v2 = dp.piece_cid_v2 + AND mdp.product = $1 + LEFT JOIN LATERAL ( + SELECT pp.task_id + FROM unnest(mdp.ref_ids) AS r(ref_id) + JOIN parked_piece_refs pr ON pr.ref_id = r.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.complete = FALSE + LIMIT 1 + ) AS t ON TRUE + LEFT JOIN harmony_task h ON h.id = t.task_id + WHERE dp.downloaded = FALSE + AND h.id IS NULL; + `, mk20.ProductNamePDPV1) + case "commp": + rows, err = tx.Query(` + SELECT dp.commp_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.commp_task_id + WHERE dp.complete = false + AND dp.downloaded = true + AND dp.commp_task_id IS NOT NULL + AND dp.after_commp = false + AND h.id IS NULL + `) + case "aggregate": + rows, err = tx.Query(` + SELECT dp.agg_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND dp.after_commp = true + AND dp.agg_task_id IS NOT NULL + AND dp.aggregated = false + AND h.id IS NULL + `) + case "add_piece": + rows, err = tx.Query(` + SELECT dp.add_piece_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.add_piece_task_id + WHERE dp.complete = false + AND dp.aggregated = true + AND dp.add_piece_task_id IS NOT NULL + AND dp.after_add_piece = false + AND h.id IS NULL + `) + case "save_cache": + rows, err = tx.Query(` + SELECT dp.save_cache_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.save_cache_task_id + WHERE dp.complete = false + AND dp.after_add_piece = true + AND dp.after_add_piece_msg = true + AND dp.save_cache_task_id IS NOT NULL + AND dp.after_save_cache = false + AND h.id IS NULL + `) + case "index": + rows, err = tx.Query(` + SELECT dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.indexing_task_id + WHERE dp.complete = false + AND dp.indexing_task_id IS NOT NULL + AND dp.after_save_cache = true + AND h.id IS NULL + `) + default: + return false, fmt.Errorf("unknown task type: %s", taskType) + } + + if err != nil { + return false, fmt.Errorf("failed to query failed tasks: %w", err) + } + defer rows.Close() + + var taskIDs []int64 + for rows.Next() { + var tid int64 + if err := rows.Scan(&tid); err != nil { + return false, fmt.Errorf("failed to scan task_id: %w", err) + } + taskIDs = append(taskIDs, tid) + } + + if err := rows.Err(); err != nil { + return false, fmt.Errorf("row iteration error: %w", err) + } + + for _, taskID := range taskIDs { + var name string + var posted time.Time + var result bool + err = tx.QueryRow(` + SELECT name, posted, result + FROM harmony_task_history + WHERE task_id = $1 + ORDER BY id DESC LIMIT 1 + `, taskID).Scan(&name, &posted, &result) + if errors.Is(err, pgx.ErrNoRows) { + // No history means can't restart this task + continue + } else if err != nil { + return false, fmt.Errorf("failed to query history: %w", err) + } + + // If result=true means the task ended successfully, no restart needed + if result { + continue + } + + log.Infow("restarting task", "task_id", taskID, "name", name) + + _, err = tx.Exec(` + INSERT INTO harmony_task (id, initiated_by, update_time, posted_time, owner_id, added_by, previous_task, name) + VALUES ($1, NULL, NOW(), $2, NULL, $3, NULL, $4) + `, taskID, posted, a.deps.MachineID, name) + if err != nil { + return false, fmt.Errorf("failed to insert harmony_task for task_id %d: %w", taskID, err) + } + } + + // All done successfully, commit the transaction + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return err + } + if !didCommit { + return fmt.Errorf("transaction did not commit") + } + + return nil +} + +func (a *WebRPC) MK20BulkRemoveFailedPDPPipelines(ctx context.Context, taskType string) error { + didCommit, err := a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + var rows *harmonydb.Query + var err error + + // We'll select pipeline fields directly based on the stage conditions + switch taskType { + case "downloading": + rows, err = tx.Query(` + SELECT + dp.id, + dp.piece_ref, + dp.commp_task_id, + dp.agg_task_id, + dp.add_piece_task_id, + dp.save_cache_task_id, + dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN market_mk20_download_pipeline mdp + ON mdp.id = dp.id + AND mdp.piece_cid_v2 = dp.piece_cid_v2 + AND mdp.product = $1 + LEFT JOIN LATERAL ( + SELECT pp.task_id + FROM unnest(mdp.ref_ids) AS r(ref_id) + JOIN parked_piece_refs pr ON pr.ref_id = r.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.task_id IS NOT NULL + LIMIT 1 + ) t ON TRUE + LEFT JOIN harmony_task h ON h.id = t.task_id + WHERE dp.complete = FALSE + AND dp.downloaded = FALSE + AND t.task_id IS NOT NULL + AND h.id IS NULL; + `, mk20.ProductNamePDPV1) + case "commp": + rows, err = tx.Query(` + SELECT dp.id, dp.piece_ref, dp.commp_task_id, dp.agg_task_id, dp.add_piece_task_id, dp.save_cache_task_id, dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.commp_task_id + WHERE dp.complete = false + AND dp.downloaded = true + AND dp.commp_task_id IS NOT NULL + AND dp.after_commp = false + AND h.id IS NULL + `) + case "aggregate": + rows, err = tx.Query(` + SELECT dp.id, dp.piece_ref, dp.commp_task_id, dp.agg_task_id, dp.add_piece_task_id, dp.save_cache_task_id, dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND after_commp = true + AND dp.agg_task_id IS NOT NULL + AND dp.aggregated = false + AND h.id IS NULL + `) + case "add_piece": + rows, err = tx.Query(` + SELECT dp.id, dp.piece_ref, dp.commp_task_id, dp.agg_task_id, dp.add_piece_task_id, dp.save_cache_task_id, dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND aggregated = true + AND dp.add_piece_task_id IS NOT NULL + AND dp.after_add_piece = false + AND h.id IS NULL + `) + case "save_cache": + rows, err = tx.Query(` + SELECT dp.id, dp.piece_ref, dp.commp_task_id, dp.agg_task_id, dp.add_piece_task_id, dp.save_cache_task_id, dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.agg_task_id + WHERE dp.complete = false + AND after_add_piece = true + AND after_add_piece_msg = true + AND dp.save_cache_task_id IS NOT NULL + AND dp.after_save_cache = false + AND h.id IS NULL + `) + case "index": + rows, err = tx.Query(` + SELECT dp.id, dp.piece_ref, dp.commp_task_id, dp.agg_task_id, dp.add_piece_task_id, dp.save_cache_task_id, dp.indexing_task_id + FROM pdp_pipeline dp + LEFT JOIN harmony_task h ON h.id = dp.indexing_task_id + WHERE dp.complete = false + AND after_save_cache = true + AND dp.indexing_task_id IS NOT NULL + AND h.id IS NULL + `) + default: + return false, fmt.Errorf("unknown task type: %s", taskType) + } + + if err != nil { + return false, fmt.Errorf("failed to query failed pipelines: %w", err) + } + defer rows.Close() + + type pipelineInfo struct { + id string + refID sql.NullInt64 + commpTaskID sql.NullInt64 + aggTaskID sql.NullInt64 + addPieceTaskID sql.NullInt64 + saveCacheTask sql.NullInt64 + indexingTaskID sql.NullInt64 + } + + var pipelines []pipelineInfo + for rows.Next() { + var p pipelineInfo + if err := rows.Scan(&p.id, &p.refID, &p.commpTaskID, &p.aggTaskID, &p.addPieceTaskID, &p.saveCacheTask, &p.indexingTaskID); err != nil { + return false, fmt.Errorf("failed to scan pdp pipeline info: %w", err) + } + pipelines = append(pipelines, p) + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("row iteration error: %w", err) + } + + for _, p := range pipelines { + // Gather task IDs + var taskIDs []int64 + if p.commpTaskID.Valid { + taskIDs = append(taskIDs, p.commpTaskID.Int64) + } + if p.aggTaskID.Valid { + taskIDs = append(taskIDs, p.aggTaskID.Int64) + } + if p.addPieceTaskID.Valid { + taskIDs = append(taskIDs, p.addPieceTaskID.Int64) + } + if p.saveCacheTask.Valid { + taskIDs = append(taskIDs, p.saveCacheTask.Int64) + } + if p.indexingTaskID.Valid { + taskIDs = append(taskIDs, p.indexingTaskID.Int64) + } + + if len(taskIDs) > 0 { + var runningTasks int + err = tx.QueryRow(`SELECT COUNT(*) FROM harmony_task WHERE id = ANY($1)`, taskIDs).Scan(&runningTasks) + if err != nil { + return false, err + } + if runningTasks > 0 { + // This should not happen if they are failed, but just in case + return false, fmt.Errorf("cannot remove deal pipeline %s: tasks are still running", p.id) + } + } + + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", p.id) // TODO: Add Correct error + + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + + _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, p.id) + if err != nil { + return false, xerrors.Errorf("failed to clean up pdp pipeline: %w", err) + } + + if p.refID.Valid { + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, p.refID.Int64) + if err != nil { + return false, fmt.Errorf("failed to remove parked_piece_refs for pipeline %s: %w", p.id, err) + } + } + + log.Infow("removed failed PDP pipeline", "id", p.id) + } + + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return err + } + if !didCommit { + return fmt.Errorf("transaction did not commit") + } + + return nil +} + +func (a *WebRPC) MK20PDPPipelineRemove(ctx context.Context, id string) error { + _, err := ulid.Parse(id) + if err != nil { + return err + } + + _, err = a.deps.DB.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var pipelines []struct { + Ref sql.NullInt64 `db:"piece_ref"` + + CommpTaskID sql.NullInt64 `db:"commp_task_id"` + AggrTaskID sql.NullInt64 `db:"agg_task_id"` + AddPieceTaskID sql.NullInt64 `db:"add_piece_task_id"` + SaveCacheTask sql.NullInt64 `db:"save_cache_task"` + IndexingTaskID sql.NullInt64 `db:"indexing_task_id"` + } + + err = tx.Select(&pipelines, `SELECT piece_ref, sector, commp_task_id, agg_task_id, indexing_task_id + FROM market_mk20_pipeline WHERE id = $1`, id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, fmt.Errorf("no deal pipeline found with id %s", id) + } + return false, err + } + + if len(pipelines) == 0 { + return false, fmt.Errorf("no deal pipeline found with id %s", id) + } + + // Collect non-null task IDs + var taskIDs []int64 + for _, pipeline := range pipelines { + if pipeline.CommpTaskID.Valid { + taskIDs = append(taskIDs, pipeline.CommpTaskID.Int64) + } + if pipeline.AggrTaskID.Valid { + taskIDs = append(taskIDs, pipeline.AggrTaskID.Int64) + } + if pipeline.AddPieceTaskID.Valid { + taskIDs = append(taskIDs, pipeline.AddPieceTaskID.Int64) + } + if pipeline.SaveCacheTask.Valid { + taskIDs = append(taskIDs, pipeline.SaveCacheTask.Int64) + } + if pipeline.IndexingTaskID.Valid { + taskIDs = append(taskIDs, pipeline.IndexingTaskID.Int64) + } + } + + // Check if any tasks are still running + if len(taskIDs) > 0 { + var runningTasks int + err = tx.QueryRow(`SELECT COUNT(*) FROM harmony_task WHERE id = ANY($1)`, taskIDs).Scan(&runningTasks) + if err != nil { + return false, err + } + if runningTasks > 0 { + return false, fmt.Errorf("cannot remove deal pipeline %s: tasks are still running", id) + } + } + + n, err := tx.Exec(`UPDATE market_mk20_deal + SET pdp_v1 = jsonb_set( + jsonb_set(pdp_v1, '{error}', to_jsonb($1::text), true), + '{complete}', to_jsonb(true), true + ) + WHERE id = $2;`, "Transaction failed", id) // TODO: Add Correct error + + if err != nil { + return false, xerrors.Errorf("failed to update market_mk20_deal: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected 1 row to be updated, got %d", n) + } + + _, err = tx.Exec(`DELETE FROM pdp_pipeline WHERE id = $1`, id) + if err != nil { + return false, xerrors.Errorf("failed to clean up pdp pipeline: %w", err) + } + + for _, pipeline := range pipelines { + if pipeline.Ref.Valid { + _, err = tx.Exec(`DELETE FROM parked_piece_refs WHERE ref_id = $1`, pipeline.Ref) + if err != nil { + return false, fmt.Errorf("failed to remove parked_piece_refs for pipeline %s: %w", id, err) + } + } + } + + return true, nil + }, harmonydb.OptionRetry()) + return err +} diff --git a/web/api/webrpc/proofshare.go b/web/api/webrpc/proofshare.go index 0809d5c8a..683cb82a3 100644 --- a/web/api/webrpc/proofshare.go +++ b/web/api/webrpc/proofshare.go @@ -193,7 +193,7 @@ func (a *WebRPC) PSListQueue(ctx context.Context) ([]*ProofShareQueueItem, error continue } - var paymentAmt string = "0" + var paymentAmt = "0" var providerID, paymentNonce int64 err := a.deps.DB.QueryRow(ctx, ` SELECT payment_cumulative_amount, provider_id, payment_nonce @@ -204,7 +204,7 @@ func (a *WebRPC) PSListQueue(ctx context.Context) ([]*ProofShareQueueItem, error return nil, xerrors.Errorf("PSListQueue: failed to query proofshare_provider_payments: %w", err) } - var prevPaymentAmt string = "0" + var prevPaymentAmt = "0" if paymentNonce > 0 { err := a.deps.DB.QueryRow(ctx, ` SELECT payment_cumulative_amount diff --git a/web/srv.go b/web/srv.go index 80f40458c..3fa9d23f7 100644 --- a/web/srv.go +++ b/web/srv.go @@ -233,7 +233,9 @@ func websocketProxy(target *url.URL, w http.ResponseWriter, r *http.Request) { http.Error(w, "Failed to connect to backend", http.StatusServiceUnavailable) return } - defer backendConn.Close() + defer func() { + _ = backendConn.Close() + }() upgrader := websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { @@ -246,7 +248,9 @@ func websocketProxy(target *url.URL, w http.ResponseWriter, r *http.Request) { http.Error(w, "Failed to upgrade connection", http.StatusInternalServerError) return } - defer clientConn.Close() + defer func() { + _ = clientConn.Close() + }() errc := make(chan error, 2) go proxyCopy(clientConn, backendConn, errc, "client -> backend") diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index 3eac0270e..d173e8289 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -70,6 +70,7 @@ class DealDetails extends LitElement {
    ${item.miner} ${item.chain_deal_id} ${item.sector}${item.offset} + ${item.offset ? item.offset : html``} + ${this.toHumanBytes(item.length)} ${this.toHumanBytes(item.raw_size)}
    ${item.miner} ${item.chain_deal_id} ${item.sector}${item.offset} + ${item.offset ? item.offset : html``} + ${this.toHumanBytes(item.length)} ${this.toHumanBytes(item.raw_size)}
    ${products?.ddo_v1 ? this.renderDDOV1(products.ddo_v1) : ''} + ${products?.pdp_v1 ? this.renderPDPV1(products.pdp_v1) : ''} ${products?.retrieval_v1 ? this.renderRetV1(products.retrieval_v1) : ''} `; } @@ -229,13 +230,30 @@ class DealDetails extends LitElement { `; } + renderPDPV1(pdp) { + if (!pdp) return ''; + return html` +
    PDP V1
    + + + + < + + + ${pdp.data_set_id ? html`` : ``} + ${pdp.piece_ids ? html`` : ``} +
    Create DataSet
    Create Piece
    Remove Piece
    Remove DataSet
    Record Keeper${pdp.record_keeper}>
    DataSet ID${pdp.data_set_id}
    Piece IDs${pdp.piece_ids}
    + `; + } + renderRetV1(ret) { if (!ret) return ''; return html`
    Retrieval v1
    - - + + +
    Indexing${ret.indexing ? 'Yes' : 'No'}
    Announce to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    Indexing${ret.indexing ? 'Yes' : 'No'}
    Announce Piece to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    Announce Payload to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    `; } diff --git a/web/static/pages/mk20/ddo-pipeline.mjs b/web/static/pages/mk20/ddo-pipeline.mjs index 001751ba7..c1ce081a3 100644 --- a/web/static/pages/mk20/ddo-pipeline.mjs +++ b/web/static/pages/mk20/ddo-pipeline.mjs @@ -40,7 +40,7 @@ class MK20DealPipelines extends LitElement { async loadData() { try { const params = [this.limit, this.offset]; - const deals = await RPCCall('MK20DealPipelines', params); + const deals = await RPCCall('MK20DDOPipelines', params); this.deals = deals || []; // Load failed tasks data diff --git a/web/static/pages/mk20/settings.mjs b/web/static/pages/mk20/settings.mjs index 5f0b2cab5..01a3bf9e4 100644 --- a/web/static/pages/mk20/settings.mjs +++ b/web/static/pages/mk20/settings.mjs @@ -34,12 +34,12 @@ class MarketManager extends LitElement { ? dataSourcesResult : Object.entries(dataSourcesResult).map(([name, enabled]) => ({ name, enabled })); - const contractsResult = await RPCCall('ListMarketContracts', []); - this.contracts = Array.isArray(contractsResult) - ? contractsResult - : Object.entries(contractsResult).map(([address, abi]) => ({ address, abi })); - - this.requestUpdate(); + // const contractsResult = await RPCCall('ListMarketContracts', []); + // this.contracts = Array.isArray(contractsResult) + // ? contractsResult + // : Object.entries(contractsResult).map(([address, abi]) => ({ address, abi })); + // + // this.requestUpdate(); } catch (err) { console.error('Failed to load data:', err); this.products = []; @@ -211,6 +211,7 @@ class MarketManager extends LitElement { + ` )} + ---> ${this.renderContractModal()} diff --git a/web/static/pages/pdp/pipeline.mjs b/web/static/pages/pdp/pipeline.mjs new file mode 100644 index 000000000..f8704717d --- /dev/null +++ b/web/static/pages/pdp/pipeline.mjs @@ -0,0 +1,374 @@ +import { LitElement, html, css } from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; +import { formatDate } from '/lib/dateutil.mjs'; + +class MK20PDPPipelines extends LitElement { + static properties = { + deals: { type: Array }, + limit: { type: Number }, + offset: { type: Number }, + totalCount: { type: Number }, + failedTasks: { type: Object }, + restartingTaskType: { type: String }, + removingTaskType: { type: String } + }; + + constructor() { + super(); + this.deals = []; + this.limit = 25; + this.offset = 0; + this.totalCount = 0; + this.failedTasks = {}; + this.restartingTaskType = ''; + this.removingTaskType = ''; + this.loadData(); + } + + connectedCallback() { + super.connectedCallback(); + // Set up an interval to update data every 5 seconds + this.intervalId = setInterval(() => this.loadData(), 5000); + } + + disconnectedCallback() { + super.disconnectedCallback(); + // Clear the interval when the element is disconnected + clearInterval(this.intervalId); + } + + async loadData() { + try { + const params = [this.limit, this.offset]; + const deals = await RPCCall('MK20PDPPipelines', params); + this.deals = deals || []; + + // Load failed tasks data + const failed = await RPCCall('MK20PipelineFailedTasks', []); + this.failedTasks = failed || {}; + + this.requestUpdate(); + } catch (error) { + console.error('Failed to load deal pipelines or failed tasks:', error); + } + } + + nextPage() { + this.offset += this.limit; + this.loadData(); + } + + prevPage() { + if (this.offset >= this.limit) { + this.offset -= this.limit; + } else { + this.offset = 0; + } + this.loadData(); + } + + renderFailedTasks() { + const { DownloadingFailed, CommPFailed, AggFailed, IndexFailed } = this.failedTasks; + const entries = []; + + const renderLine = (label, count, type) => { + const isRestarting = this.restartingTaskType === type; + const isRemoving = this.removingTaskType === type; + const isWorking = isRestarting || isRemoving; + return html` +
    + ${label} Task: ${count} +
    + + ${isWorking ? 'Working...' : 'Actions'} + + + +
    +
    + `; + }; + + if (DownloadingFailed > 0) { + entries.push(renderLine('Downloading', DownloadingFailed, 'downloading')); + } + if (CommPFailed > 0) { + entries.push(renderLine('CommP', CommPFailed, 'commp')); + } + if (AggFailed > 0) { + entries.push(renderLine('Aggregate', AggFailed, 'aggregate')); + } + if (IndexFailed > 0) { + entries.push(renderLine('Index', IndexFailed, 'index')); + } + + if (entries.length === 0) { + return null; + } + + return html` +
    +

    Failed Tasks

    + ${entries} +
    + `; + } + + async restartFailedTasks(type) { + this.restartingTaskType = type; + this.removingTaskType = ''; + this.requestUpdate(); + + try { + await RPCCall('MK20BulkRestartFailedMarketTasks', [type]); + await this.loadData(); + } catch (err) { + console.error('Failed to restart tasks:', err); + alert(`Failed to restart ${type} tasks: ${err.message || err}`); + } finally { + this.restartingTaskType = ''; + this.requestUpdate(); + } + } + + async removeFailedPipelines(type) { + this.removingTaskType = type; + this.restartingTaskType = ''; + this.requestUpdate(); + + try { + await RPCCall('MK20BulkRemoveFailedMarketPipelines', [type]); + await this.loadData(); + } catch (err) { + console.error('Failed to remove pipelines:', err); + alert(`Failed to remove ${type} pipelines: ${err.message || err}`); + } finally { + this.removingTaskType = ''; + this.requestUpdate(); + } + } + + render() { + return html` + + + +
    + ${this.renderFailedTasks()} +

    + Deal Pipelines + +

    + + + + + + + + + + + + ${this.deals.map( + (deal) => html` + + + + + + + + ` + )} + +
    Created AtUUIDSP IDPiece CIDStatus
    ${formatDate(deal.created_at)} + ${deal.id} + ${deal.miner} + ${this.formatPieceCid(deal.piece_cid_v2)} + ${this.getDealStatus(deal)}
    +
    + + Page ${(this.offset / this.limit) + 1} + +
    +
    + `; + } + + formatPieceCid(pieceCid) { + if (!pieceCid) return ''; + if (pieceCid.length <= 24) { + return pieceCid; + } + const start = pieceCid.substring(0, 16); + const end = pieceCid.substring(pieceCid.length - 8); + return `${start}...${end}`; + } + + formatBytes(bytes) { + const units = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + let size = bytes; + while (size >= 1024 && i < units.length - 1) { + size /= 1024; + i++; + } + if (i === 0) { + return `${size} ${units[i]}`; + } else { + return `${size.toFixed(2)} ${units[i]}`; + } + } + + getDealStatus(deal) { + if (deal.complete) { + return '(#########) Complete'; + } else if (!deal.complete && deal.announce && deal.indexed) { + return '(########.) Announcing'; + } else if (deal.sealed && !deal.indexed) { + return '(#######..) Indexing'; + } else if (deal.sector?.Valid && !deal.sealed) { + return '(######...) Sealing'; + } else if (deal.aggregated && !deal.sector?.Valid) { + return '(#####....) Assigning Sector'; + } else if (deal.after_commp && !deal.aggregated) { + return '(####.....) Aggregating Deal'; + } else if (deal.downloaded && !deal.after_commp) { + return '(###......) CommP'; + } else if (deal.started && !deal.downloaded) { + return '(##.......) Downloading'; + } else { + return '(#........) Accepted'; + } + } + + static styles = css` + .pagination-controls { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + + .info-btn { + position: relative; + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + font-size: 1em; + margin-left: 8px; + } + + .tooltip-text { + display: none; + position: absolute; + top: 50%; + left: 120%; + transform: translateY(-50%); + min-width: 440px; + max-width: 600px; + background-color: #333; + color: #fff; + padding: 8px; + border-radius: 4px; + font-size: 0.8em; + text-align: left; + white-space: normal; + z-index: 10; + } + + .info-btn:hover .tooltip-text { + display: block; + } + + .copy-btn { + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + padding: 0 0 0 5px; + } + + .copy-btn svg { + vertical-align: middle; + } + + .copy-btn:hover { + color: #0d6efd; + } + + .failed-tasks { + margin-bottom: 1rem; + } + .failed-tasks h2 { + margin: 0 0 0.5rem 0; + } + + details > summary { + display: inline-block; + cursor: pointer; + outline: none; + } + + .btn { + margin: 0 4px; + } + `; +} + +customElements.define('mk20-pdp-pipelines', MK20PDPPipelines); diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index 1c4c8d1fa..fa7a75ac3 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -384,89 +384,169 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { `; } })()} - ${entry.mk20_pipeline ? html` -
    PIPELINE ACTIVE
    + ${entry.mk20_ddo_pipeline ? html` +
    DDO PIPELINE ACTIVE
    Controls - Created At${formatDate(entry.mk20_pipeline.created_at)} - Piece CID${entry.mk20_pipeline.piece_cid} - Piece Size${this.toHumanBytes(entry.mk20_pipeline.piece_size)} - Raw Size${entry.mk20_pipeline.raw_size.Valid ? this.toHumanBytes(entry.mk20_pipeline.raw_size.Int64) : 'N/A'} - Offline - URL${entry.mk20_pipeline.url.Valid ? entry.mk20_pipeline.url.String : 'N/A'} - Headers
    ${JSON.stringify(entry.mk20_pipeline.headers, null, 2)}
    - Should Index${this.renderNullableYesNo(entry.mk20_pipeline.indexing)} + Created At${formatDate(entry.mk20_ddo_pipeline.created_at)} + Piece CID${entry.mk20_ddo_pipeline.piece_cid} + Piece Size${this.toHumanBytes(entry.mk20_ddo_pipeline.piece_size)} + Raw Size${entry.mk20_ddo_pipeline.raw_size.Valid ? this.toHumanBytes(entry.mk20_ddo_pipeline.raw_size.Int64) : 'N/A'} + Offline + URL${entry.mk20_ddo_pipeline.url.Valid ? entry.mk20_ddo_pipeline.url.String : 'N/A'} + Headers
    ${JSON.stringify(entry.mk20_ddo_pipeline.headers, null, 2)}
    + Should Index${this.renderNullableYesNo(entry.mk20_ddo_pipeline.indexing)} Announce - ${this.renderNullableYesNo(entry.mk20_pipeline.announce)} + ${this.renderNullableYesNo(entry.mk20_ddo_pipeline.announce)}
    Progress šŸ› ļø
    Data Fetched - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.downloaded)} + ${this.renderNullableDoneNotDone(entry.mk20_ddo_pipeline.downloaded)} After Commp - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.after_commp)} + ${this.renderNullableDoneNotDone(entry.mk20_ddo_pipeline.after_commp)} Aggregated - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.aggregated)} + ${this.renderNullableDoneNotDone(entry.mk20_ddo_pipeline.aggregated)} Sealed - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.sealed)} + ${this.renderNullableDoneNotDone(entry.mk20_ddo_pipeline.sealed)} Indexed - ${this.renderNullableDoneNotDone(entry.mk20_pipeline.indexed)} + ${this.renderNullableDoneNotDone(entry.mk20_ddo_pipeline.indexed)} Announced - +
    Early States šŸŒæ
    Commp Task ID - ${entry.mk20_pipeline.commp_task_id.Valid - ? html`` - : 'N/A'} + ${entry.mk20_ddo_pipeline.commp_task_id.Valid ? html`` : 'N/A'} Aggregation Task ID - ${entry.mk20_pipeline.agg_task_id.Valid - ? html`` - : 'N/A'} + ${entry.mk20_ddo_pipeline.agg_task_id.Valid ? html`` : 'N/A'}
    Sealing šŸ“¦
    - Sector${entry.mk20_pipeline.sector.Valid ? html`${entry.mk20_pipeline.sector.Int64}` : 'N/A'} - Reg Seal Proof${entry.mk20_pipeline.reg_seal_proof.Valid ? entry.mk20_pipeline.reg_seal_proof.Int64 : 'N/A'} - Sector Offset${entry.mk20_pipeline.sector_offset.Valid ? entry.mk20_pipeline.sector_offset.Int64 : 'N/A'} + Sector${entry.mk20_ddo_pipeline.sector.Valid ? html`${entry.mk20_ddo_pipeline.sector.Int64}` : 'N/A'} + Reg Seal Proof${entry.mk20_ddo_pipeline.reg_seal_proof.Valid ? entry.mk20_ddo_pipeline.reg_seal_proof.Int64 : 'N/A'} + Sector Offset${entry.mk20_ddo_pipeline.sector_offset.Valid ? entry.mk20_ddo_pipeline.sector_offset.Int64 : 'N/A'}
    Indexing šŸ”
    - Indexing Created At${entry.mk20_pipeline.indexing_created_at.Valid ? formatDate(entry.mk20_pipeline.indexing_created_at.Time) : 'N/A'} + Indexing Created At${entry.mk20_ddo_pipeline.indexing_created_at.Valid ? formatDate(entry.mk20_ddo_pipeline.indexing_created_at.Time) : 'N/A'} Indexing Task ID - ${entry.mk20_pipeline.indexing_task_id.Valid - ? html`` - : 'N/A'} + ${entry.mk20_ddo_pipeline.indexing_task_id.Valid ? html`` : 'N/A'} - ` : html` - No Pipeline Data - `} + ` : html`No DDO Pipeline Data`} + ${entry.mk20_pdp_pipeline ? html` +
    PDP PIPELINE ACTIVE
    + + Controls + + + + + Created At${formatDate(entry.mk20_pdp_pipeline.created_at)} + Piece CID${entry.mk20_pdp_pipeline.piece_cid_v2} + Should Index${this.renderNullableYesNo(entry.mk20_pdp_pipeline.indexing)} + + Announce Piece + ${this.renderNullableYesNo(entry.mk20_pdp_pipeline.announce)} + + + Announce Payload + ${this.renderNullableYesNo(entry.mk20_pdp_pipeline.announce_payload)} + + +
    Progress šŸ› ļø
    + + Data Fetched + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.downloaded)} + + + After Commp + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.after_commp)} + + + Aggregated + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.aggregated)} + + + Add Piece + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.after_add_piece)} + + + Add Piece Success + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.after_add_piece_msg)} + + + Save Cache + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.after_save_cache)} + + + Indexed + ${this.renderNullableDoneNotDone(entry.mk20_pdp_pipeline.indexed)} + + + Announced + + + +
    Early States šŸŒæ
    + + Commp Task ID + + ${entry.mk20_pdp_pipeline.commp_task_id.Valid ? html`` : 'N/A'} + + + + Aggregation Task ID + + ${entry.mk20_pdp_pipeline.agg_task_id.Valid ? html`` : 'N/A'} + + + + Add Piece Task ID + + ${entry.mk20_pdp_pipeline.add_piece_task_id.Valid ? html`` : 'N/A'} + + + + Save Cache Task ID + + ${entry.mk20_pdp_pipeline.save_cache_task_id.Valid ? html`` : 'N/A'} + + +
    Indexing šŸ”
    + Indexing Created At${entry.mk20_pdp_pipeline.indexing_created_at.Valid ? formatDate(entry.mk20_pdp_pipeline.indexing_created_at.Time) : 'N/A'} + + Indexing Task ID + + ${entry.mk20_pdp_pipeline.indexing_task_id.Valid ? html`` : 'N/A'} + + + ` : html`No PDP Pipeline Data`} `)} From 1863ea198a5f09aef16e9f98428c074367bd4fec Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 9 Sep 2025 01:21:11 +0400 Subject: [PATCH 38/55] undo m20 ddo download --- market/mk20/types_test.go | 4 +- tasks/storage-market/mk20.go | 141 ++++++++++----------- web/api/webrpc/market_2.go | 8 +- web/static/pages/pdp/index.html | 46 ++++--- web/static/pages/pdp/pdp_deals.mjs | 191 +++++++++++++++++++++++++++++ web/static/pages/pdp/pipeline.mjs | 26 ++-- 6 files changed, 316 insertions(+), 100 deletions(-) create mode 100644 web/static/pages/pdp/pdp_deals.mjs diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go index 150bf4093..af5a42171 100644 --- a/market/mk20/types_test.go +++ b/market/mk20/types_test.go @@ -281,8 +281,8 @@ func TestDeal_Products_OmitEmptyInnerFields(t *testing.T) { } } -func TestManualUnMarshal(t *testing.T) { - iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdp_v1\":{\"create_data_Set\":true,\"add_piece\":true,\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" +func TestPartialUnmarshal(t *testing.T) { + iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdpV1\":{\"createDataSet\":true,\"addPiece\":true,\"recordKeeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" var deal Deal if err := json.Unmarshal([]byte(iString), &deal); err != nil { t.Fatal(err) diff --git a/tasks/storage-market/mk20.go b/tasks/storage-market/mk20.go index 97577d744..e0bd1c203 100644 --- a/tasks/storage-market/mk20.go +++ b/tasks/storage-market/mk20.go @@ -83,7 +83,7 @@ func (d *CurioStorageDealMarket) processMK20Deals(ctx context.Context) { } }() d.processMK20DealPieces(ctx) - d.downloadMk20Deal(ctx) + //d.downloadMk20Deal(ctx) d.processMK20DealAggregation(ctx) d.processMK20DealIngestion(ctx) } @@ -603,12 +603,12 @@ func (d *CurioStorageDealMarket) processMK20DealPieces(ctx context.Context) { } func (d *CurioStorageDealMarket) processMk20Pieces(ctx context.Context, piece MK20PipelinePiece) error { - //err := d.downloadMk20Deal(ctx, piece) - //if err != nil { - // return err - //} + err := d.downloadMk20Deal(ctx, piece) + if err != nil { + return err + } - err := d.findOfflineURLMk20Deal(ctx, piece) + err = d.findOfflineURLMk20Deal(ctx, piece) if err != nil { return err } @@ -629,72 +629,73 @@ func (d *CurioStorageDealMarket) processMk20Pieces(ctx context.Context, piece MK // downloadMk20Deal handles the downloading process of an MK20 pipeline piece by scheduling it in the database and updating its status. // If the pieces are part of an aggregation deal then we download for short term otherwise, // we download for long term to avoid the need to have unsealed copy -func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context) { - n, err := d.db.Exec(ctx, `SELECT mk20_ddo_mark_downloaded($1)`, mk20.ProductNameDDOV1) - if err != nil { - log.Errorf("failed to mark PDP downloaded piece: %v", err) - - } - log.Debugf("Succesfully marked %d PDP pieces as downloaded", n) - - //if !piece.Downloaded && piece.Started { - //_, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { - // var refid int64 - // err = tx.QueryRow(`SELECT u.ref_id FROM ( - // SELECT unnest(dp.ref_ids) AS ref_id - // FROM market_mk20_download_pipeline dp - // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - // ) u - // JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id - // JOIN parked_pieces pp ON pp.id = pr.piece_id - // WHERE pp.complete = TRUE - // LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&refid) - // if err != nil { - // if errors.Is(err, pgx.ErrNoRows) { - // return false, nil - // } - // return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) - // } - // - // // Remove other ref_ids from piece_park_refs - // _, err = tx.Exec(`DELETE FROM parked_piece_refs - // WHERE ref_id IN ( - // SELECT unnest(dp.ref_ids) - // FROM market_mk20_download_pipeline dp - // WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 - // ) - // AND ref_id != $5;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1, refid) - // if err != nil { - // return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) - // } - // - // _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, - // piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1) - // if err != nil { - // return false, xerrors.Errorf("failed to delete piece from download table: %w", err) - // } - // - // pieceIDUrl := url.URL{ - // Scheme: "pieceref", - // Opaque: fmt.Sprintf("%d", refid), - // } - // - // _, err = tx.Exec(`UPDATE market_mk20_pipeline SET downloaded = TRUE, url = $1 - // WHERE id = $2 - // AND piece_cid = $3 - // AND piece_size = $4`, - // pieceIDUrl.String(), piece.ID, piece.PieceCID, piece.PieceSize) - // if err != nil { - // return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) - // } - // piece.Downloaded = true - // return true, nil - //}, harmonydb.OptionRetry()) - // +func (d *CurioStorageDealMarket) downloadMk20Deal(ctx context.Context, piece MK20PipelinePiece) error { + //n, err := d.db.Exec(ctx, `SELECT mk20_ddo_mark_downloaded($1)`, mk20.ProductNameDDOV1) //if err != nil { - // return xerrors.Errorf("failed to schedule the deal for download: %w", err) - //} + // log.Errorf("failed to mark PDP downloaded piece: %v", err) + // //} + //log.Debugf("Succesfully marked %d PDP pieces as downloaded", n) + + if !piece.Downloaded && piece.Started { + _, err := d.db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + var refid int64 + err = tx.QueryRow(`SELECT u.ref_id FROM ( + SELECT unnest(dp.ref_ids) AS ref_id + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + ) u + JOIN parked_piece_refs pr ON pr.ref_id = u.ref_id + JOIN parked_pieces pp ON pp.id = pr.piece_id + WHERE pp.complete = TRUE + LIMIT 1;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1).Scan(&refid) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, xerrors.Errorf("failed to check if the piece is downloaded: %w", err) + } + + // Remove other ref_ids from piece_park_refs + _, err = tx.Exec(`DELETE FROM parked_piece_refs + WHERE ref_id IN ( + SELECT unnest(dp.ref_ids) + FROM market_mk20_download_pipeline dp + WHERE dp.id = $1 AND dp.piece_cid = $2 AND dp.piece_size = $3 AND dp.product = $4 + ) + AND ref_id != $5;`, piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1, refid) + if err != nil { + return false, xerrors.Errorf("failed to remove other ref_ids from piece_park_refs: %w", err) + } + + _, err = tx.Exec(`DELETE FROM market_mk20_download_pipeline WHERE id = $1 AND piece_cid = $2 AND piece_size = $3 AND product = $4;`, + piece.ID, piece.PieceCID, piece.PieceSize, mk20.ProductNameDDOV1) + if err != nil { + return false, xerrors.Errorf("failed to delete piece from download table: %w", err) + } + + pieceIDUrl := url.URL{ + Scheme: "pieceref", + Opaque: fmt.Sprintf("%d", refid), + } + + _, err = tx.Exec(`UPDATE market_mk20_pipeline SET downloaded = TRUE, url = $1 + WHERE id = $2 + AND piece_cid = $3 + AND piece_size = $4`, + pieceIDUrl.String(), piece.ID, piece.PieceCID, piece.PieceSize) + if err != nil { + return false, xerrors.Errorf("failed to update pipeline piece table: %w", err) + } + piece.Downloaded = true + return true, nil + }, harmonydb.OptionRetry()) + + if err != nil { + return xerrors.Errorf("failed to schedule the deal for download: %w", err) + } + } + return nil } // findOfflineURLMk20Deal find the URL for offline piece. In MK20, we don't work directly with remote pieces, we download them diff --git a/web/api/webrpc/market_2.go b/web/api/webrpc/market_2.go index 51020a6ce..e471454d9 100644 --- a/web/api/webrpc/market_2.go +++ b/web/api/webrpc/market_2.go @@ -949,8 +949,8 @@ type MK20PDPPipelineFailedStats struct { DownloadingFailed int64 CommPFailed int64 AggFailed int64 - AddPiece int64 - SaveCache int64 + AddPieceFailed int64 + SaveCacheFailed int64 IndexFailed int64 } @@ -1080,8 +1080,8 @@ func (a *WebRPC) MK20PDPPipelineFailedTasks(ctx context.Context) (*MK20PDPPipeli DownloadingFailed: counts.DownloadingFailed, CommPFailed: counts.CommPFailed, AggFailed: counts.AggFailed, - AddPiece: counts.AddPieceFailed, - SaveCache: counts.SaveCacheFailed, + AddPieceFailed: counts.AddPieceFailed, + SaveCacheFailed: counts.SaveCacheFailed, IndexFailed: counts.IndexFailed, }, nil } diff --git a/web/static/pages/pdp/index.html b/web/static/pages/pdp/index.html index 049237a94..b982e7b77 100644 --- a/web/static/pages/pdp/index.html +++ b/web/static/pages/pdp/index.html @@ -1,24 +1,42 @@ - Node Info + PDP Overview + + + - -
    -
    -
    -

    Proof of Data Possession

    + +
    +
    +
    +

    Proof of Data Possession

    +
    +
    +
    +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    +
    + +
    +
    +
    - -
    -
    - -
    -
    -
    - + diff --git a/web/static/pages/pdp/pdp_deals.mjs b/web/static/pages/pdp/pdp_deals.mjs new file mode 100644 index 000000000..aaf163b61 --- /dev/null +++ b/web/static/pages/pdp/pdp_deals.mjs @@ -0,0 +1,191 @@ +import {css, html, LitElement} from 'https://cdn.jsdelivr.net/gh/lit/dist@3/all/lit-all.min.js'; +import RPCCall from '/lib/jsonrpc.mjs'; +import { formatDate } from '/lib/dateutil.mjs'; +import '/ux/yesno.mjs'; + +class MK20PDPDealList extends LitElement { + static properties = { + deals: { type: Array }, + limit: { type: Number }, + offset: { type: Number }, + totalCount: { type: Number }, + }; + + constructor() { + super(); + this.deals = []; + this.limit = 25; + this.offset = 0; + this.totalCount = 0; + this.loadData(); + } + + async loadData() { + try { + const params = [this.limit, this.offset]; + this.deals = await RPCCall('MK20PDPStorageDeals', params); + this.requestUpdate(); + } catch (error) { + console.error('Failed to load pdp deals:', error); + } + } + + nextPage() { + this.offset += this.limit; + this.loadData(); + } + + prevPage() { + if (this.offset >= this.limit) { + this.offset -= this.limit; + } else { + this.offset = 0; + } + this.loadData(); + } + + render() { + // Check if there's an error or if the deals array is empty + if (!this.deals || this.deals.length === 0) { + return html``; // Return an empty template if there's no data to render + } + + return html` + + + +
    +

    +

    PDP Deal List + +

    + + + + + + + + + + + + + + + ${this.deals.map( + (deal) => html` + + + + + + + + + ` + )} + +
    Created AtIDProviderPiece CIDPiece SizeProcessedError
    ${formatDate(deal.created_at)}${deal.id}${deal.miner.Valid ? deal.miner.String : '-'} + ${deal.piece_cid_v2 + ? html`${this.formatPieceCid(deal.piece_cid_v2.String)}` + : 'Not Available'} +
    +
    + + Page ${(this.offset / this.limit) + 1} + +
    +
    + `; + } + + formatBytes(bytes) { + const units = ['Bytes', 'KiB', 'MiB', 'GiB', 'TiB']; + let i = 0; + let size = bytes; + while (size >= 1024 && i < units.length - 1) { + size /= 1024; + i++; + } + if (i === 0) { + return `${size} ${units[i]}`; + } else { + return `${size.toFixed(2)} ${units[i]}`; + } + } + + formatPieceCid(pieceCid) { + if (!pieceCid) return ''; + if (pieceCid.length <= 24) { + return pieceCid; + } + const start = pieceCid.substring(0, 16); + const end = pieceCid.substring(pieceCid.length - 8); + return `${start}...${end}`; + } + + static styles = css` + .pagination-controls { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + } + + .info-btn { + position: relative; + border: none; + background: transparent; + cursor: pointer; + color: #17a2b8; + font-size: 1em; + margin-left: 8px; + } + + .tooltip-text { + display: none; + position: absolute; + top: 50%; + left: 120%; /* Position the tooltip to the right of the button */ + transform: translateY(-50%); /* Center the tooltip vertically */ + min-width: 440px; + max-width: 600px; + background-color: #333; + color: #fff; + padding: 8px; + border-radius: 4px; + font-size: 0.8em; + text-align: left; + white-space: normal; + z-index: 10; + } + + .info-btn:hover .tooltip-text { + display: block; + } + `; +} + +customElements.define('mk20-pdp-deal-list', MK20PDPDealList); \ No newline at end of file diff --git a/web/static/pages/pdp/pipeline.mjs b/web/static/pages/pdp/pipeline.mjs index f8704717d..b9025f34f 100644 --- a/web/static/pages/pdp/pipeline.mjs +++ b/web/static/pages/pdp/pipeline.mjs @@ -44,7 +44,7 @@ class MK20PDPPipelines extends LitElement { this.deals = deals || []; // Load failed tasks data - const failed = await RPCCall('MK20PipelineFailedTasks', []); + const failed = await RPCCall('MK20PDPPipelineFailedTasks', []); this.failedTasks = failed || {}; this.requestUpdate(); @@ -68,7 +68,7 @@ class MK20PDPPipelines extends LitElement { } renderFailedTasks() { - const { DownloadingFailed, CommPFailed, AggFailed, IndexFailed } = this.failedTasks; + const { DownloadingFailed, CommPFailed, AggFailed, AddPieceFailed, SaveCacheFailed, IndexFailed } = this.failedTasks; const entries = []; const renderLine = (label, count, type) => { @@ -110,6 +110,12 @@ class MK20PDPPipelines extends LitElement { if (AggFailed > 0) { entries.push(renderLine('Aggregate', AggFailed, 'aggregate')); } + if (AddPieceFailed > 0) { + entries.push(renderLine('AddPiece', AggFailed, 'add_piece')); + } + if (SaveCacheFailed > 0) { + entries.push(renderLine('SaveCache', AggFailed, 'save_cache')); + } if (IndexFailed > 0) { entries.push(renderLine('Index', IndexFailed, 'index')); } @@ -132,7 +138,7 @@ class MK20PDPPipelines extends LitElement { this.requestUpdate(); try { - await RPCCall('MK20BulkRestartFailedMarketTasks', [type]); + await RPCCall('MK20BulkRestartFailedPDPTasks', [type]); await this.loadData(); } catch (err) { console.error('Failed to restart tasks:', err); @@ -149,7 +155,7 @@ class MK20PDPPipelines extends LitElement { this.requestUpdate(); try { - await RPCCall('MK20BulkRemoveFailedMarketPipelines', [type]); + await RPCCall('MK20BulkRemoveFailedPDPPipelines', [type]); await this.loadData(); } catch (err) { console.error('Failed to remove pipelines:', err); @@ -172,7 +178,7 @@ class MK20PDPPipelines extends LitElement {
    ${this.renderFailedTasks()}

    - Deal Pipelines + PDP Pipelines

    -
    diff --git a/web/static/pages/pdp/index.html b/web/static/pages/pdp/index.html index b982e7b77..bcbd23694 100644 --- a/web/static/pages/pdp/index.html +++ b/web/static/pages/pdp/index.html @@ -1,42 +1,41 @@ - - PDP Overview - - - - - - - - -
    -
    -
    -

    Proof of Data Possession

    -
    -
    -
    -
    - -
    -
    -
    -
    -
    - + + PDP Overview + + + + + + + + +
    +
    +
    +

    Proof of Data Possession

    -
    -
    -
    -
    -
    -
    - +
    +
    +
    +
    - -
    - - - + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + +
    +
    +
    + + + diff --git a/web/static/pages/pdp/pdp_deals.mjs b/web/static/pages/pdp/pdp_deals.mjs index aaf163b61..e9971f10c 100644 --- a/web/static/pages/pdp/pdp_deals.mjs +++ b/web/static/pages/pdp/pdp_deals.mjs @@ -78,9 +78,7 @@ class MK20PDPDealList extends LitElement { Created At ID - Provider Piece CID - Piece Size Processed Error @@ -91,7 +89,6 @@ class MK20PDPDealList extends LitElement { ${formatDate(deal.created_at)} ${deal.id} - ${deal.miner.Valid ? deal.miner.String : '-'} ${deal.piece_cid_v2 ? html`${this.formatPieceCid(deal.piece_cid_v2.String)}` diff --git a/web/static/pages/pdp/pipeline.mjs b/web/static/pages/pdp/pipeline.mjs index b9025f34f..764519a8c 100644 --- a/web/static/pages/pdp/pipeline.mjs +++ b/web/static/pages/pdp/pipeline.mjs @@ -176,6 +176,7 @@ class MK20PDPPipelines extends LitElement {
    +

    ${this.renderFailedTasks()}

    PDP Pipelines diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index fa7a75ac3..f2f0e9ad4 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -134,9 +134,10 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { ${item.boost_deal ? 'Boost' : (item.legacy_deal ? 'Legacy' : 'DDO')} ${item.miner} ${item.chain_deal_id} - ${item.sector} - ${item.offset ? item.offset : html``} + ${item.sector > 0 ? html`${item.sector}`: "NA" } + + ${item.offset.Valid ? item.offset.int64 : html`NA`} ${this.toHumanBytes(item.length)} ${this.toHumanBytes(item.raw_size)} @@ -345,7 +346,8 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { Piece Size${this.toHumanBytes(entry.deal.deal.data.piece_size)}

    Status šŸŸ¢ļøšŸ”“
    - Error${entry.deal.error.Valid ? entry.deal.error.String : 'N/A'} + DDO Error${entry.deal.ddoerr.Valid ? entry.deal.ddoerr.String : 'N/A'} + PDP Error${entry.deal.pdperr.Valid ? entry.deal.pdperr.String : 'N/A'} ${(() => { const matchingPieceDeals = this.data.deals.filter(deal => deal.id === entry.deal.uuid); if (matchingPieceDeals.length > 0) { @@ -393,12 +395,7 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement { Created At${formatDate(entry.mk20_ddo_pipeline.created_at)} - Piece CID${entry.mk20_ddo_pipeline.piece_cid} - Piece Size${this.toHumanBytes(entry.mk20_ddo_pipeline.piece_size)} - Raw Size${entry.mk20_ddo_pipeline.raw_size.Valid ? this.toHumanBytes(entry.mk20_ddo_pipeline.raw_size.Int64) : 'N/A'} - Offline - URL${entry.mk20_ddo_pipeline.url.Valid ? entry.mk20_ddo_pipeline.url.String : 'N/A'} - Headers
    ${JSON.stringify(entry.mk20_ddo_pipeline.headers, null, 2)}
    + Piece CID${entry.mk20_ddo_pipeline.piece_cid_v2} Should Index${this.renderNullableYesNo(entry.mk20_ddo_pipeline.indexing)} Announce From 4cf1fb51afdf09bdefc24177c6e8dddfcdc9a634 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 9 Sep 2025 20:42:06 +0400 Subject: [PATCH 42/55] fix test --- documentation/en/curio-cli/sptool.md | 16 ++-- extern/filecoin-ffi | 2 +- go.mod | 53 +++++++------ go.sum | 108 +++++++++++++-------------- lib/ffi/sdr_funcs.go | 16 +--- lib/ffi/snap_funcs.go | 9 +-- market/mk20/client/client.go | 8 +- market/mk20/types_test.go | 2 +- tasks/pdp/task_commp.go | 6 +- 9 files changed, 101 insertions(+), 119 deletions(-) diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 8e2c03419..13750bd7e 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -973,18 +973,18 @@ USAGE: OPTIONS: --http-url value http url to CAR file --http-headers value [ --http-headers value ] http headers to be passed with the request (e.g key=value) - --provider value storage provider on-chain address + --provider value PDP providers's URL --pcidv2 value pcidv2 of the CAR file --wallet value wallet address to be used to initiate the deal --aggregate value aggregate file path for the deal --put used HTTP put as data source (default: false) - --add-root add root (default: false) - --add-proofset add proofset (default: false) - --remove-root remove root (default: false) - --remove-proofset remove proofset (default: false) + --add-piece add piece (default: false) + --add-dataset add dataset (default: false) + --remove-piece remove piece (default: false) + --remove-dataset remove dataset (default: false) --record-keeper value record keeper address - --root-id value [ --root-id value ] root IDs - --proofset-id value proofset IDs (default: 0) + --piece-id value [ --piece-id value ] root IDs + --dataset-id value dataset IDs (default: 0) --help, -h show help ``` @@ -1012,7 +1012,7 @@ USAGE: sptool toolbox mk20-client upload [command options] OPTIONS: - --provider value storage provider on-chain address + --provider value PDP providers's URL --deal value deal id to upload to --help, -h show help ``` diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 552ab5c27..44f5dc459 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 552ab5c27e6bd909f7fbf5c079d0f58b789c3e6f +Subproject commit 44f5dc459be3b74aec77138c4d3e976324b0d17b diff --git a/go.mod b/go.mod index 3435e314b..df21d54ee 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23.10 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 - github.com/BurntSushi/toml v1.4.0 + github.com/BurntSushi/toml v1.5.0 github.com/CAFxX/httpcompression v0.0.9 github.com/KarpelesLab/reflink v1.0.1 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 @@ -14,8 +14,8 @@ require ( github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/docker/go-units v0.5.0 github.com/dustin/go-humanize v1.0.1 - github.com/elastic/go-sysinfo v1.7.0 - github.com/etclabscore/go-openrpc-reflect v0.0.36 + github.com/elastic/go-sysinfo v1.15.4 + github.com/etclabscore/go-openrpc-reflect v0.0.37 github.com/ethereum/go-ethereum v1.15.0 github.com/fatih/color v1.18.0 github.com/filecoin-project/filecoin-ffi v1.33.1-dev @@ -26,20 +26,20 @@ require ( github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-data-segment v0.0.1 - github.com/filecoin-project/go-f3 v0.8.9 + github.com/filecoin-project/go-f3 v0.8.10 github.com/filecoin-project/go-fil-commcid v0.3.1 github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 - github.com/filecoin-project/go-jsonrpc v0.7.1 + github.com/filecoin-project/go-jsonrpc v0.8.0 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-state-types v0.17.0-dev2 github.com/filecoin-project/go-statestore v0.2.0 - github.com/filecoin-project/lotus v1.33.1 + github.com/filecoin-project/lotus v1.34.0-rc2 github.com/filecoin-project/specs-actors/v2 v2.3.6 github.com/filecoin-project/specs-actors/v5 v5.0.6 github.com/filecoin-project/specs-actors/v6 v6.0.2 github.com/filecoin-project/specs-actors/v7 v7.0.1 github.com/gbrlsnchs/jwt/v3 v3.0.1 - github.com/georgysavva/scany/v2 v2.1.3 + github.com/georgysavva/scany/v2 v2.1.4 github.com/go-chi/chi/v5 v5.2.2 github.com/go-chi/httprate v0.15.0 github.com/golang-jwt/jwt/v4 v4.5.2 @@ -67,7 +67,7 @@ require ( github.com/ipfs/go-log/v2 v2.6.0 github.com/ipld/frisbii v0.6.1 github.com/ipld/go-car v0.6.2 - github.com/ipld/go-car/v2 v2.14.3 + github.com/ipld/go-car/v2 v2.15.0 github.com/ipld/go-ipld-prime v0.21.0 github.com/ipni/go-libipni v0.6.19 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 @@ -81,7 +81,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.16.0 + github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.2 github.com/multiformats/go-multihash v0.2.3 @@ -90,7 +90,7 @@ require ( github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 - github.com/puzpuzpuz/xsync/v2 v2.4.0 + github.com/puzpuzpuz/xsync/v2 v2.5.1 github.com/raulk/clock v1.1.0 github.com/samber/lo v1.47.0 github.com/schollz/progressbar/v3 v3.18.0 @@ -107,14 +107,14 @@ require ( go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.39.0 + golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b - golang.org/x/net v0.41.0 - golang.org/x/sync v0.15.0 - golang.org/x/sys v0.33.0 - golang.org/x/term v0.32.0 - golang.org/x/text v0.26.0 - golang.org/x/tools v0.34.0 + golang.org/x/net v0.42.0 + golang.org/x/sync v0.16.0 + golang.org/x/sys v0.35.0 + golang.org/x/term v0.34.0 + golang.org/x/text v0.28.0 + golang.org/x/tools v0.35.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da ) @@ -162,7 +162,7 @@ require ( github.com/drand/kyber v1.3.1 // indirect github.com/drand/kyber-bls12381 v0.3.3 // indirect github.com/elastic/go-elasticsearch/v7 v7.14.0 // indirect - github.com/elastic/go-windows v1.0.0 // indirect + github.com/elastic/go-windows v1.0.2 // indirect github.com/elastic/gosigar v0.14.3 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect @@ -247,7 +247,6 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect @@ -274,7 +273,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mattn/go-sqlite3 v1.14.32 // indirect github.com/miekg/dns v1.1.66 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -346,7 +345,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect github.com/zondax/hid v0.9.2 // indirect - github.com/zondax/ledger-filecoin-go v1.0.1 // indirect + github.com/zondax/ledger-filecoin-go v1.1.0 // indirect github.com/zondax/ledger-go v1.0.0 // indirect github.com/zyedidia/generic v1.2.1 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect @@ -359,8 +358,8 @@ require ( go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.50.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.34.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect @@ -368,12 +367,12 @@ require ( go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/mod v0.25.0 // indirect + golang.org/x/mod v0.26.0 // indirect golang.org/x/time v0.12.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250212204824-5a70512c5d8b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b // indirect - google.golang.org/grpc v1.70.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/grpc v1.73.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index ad8f66c03..4fed7203f 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,8 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg= github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM= @@ -259,10 +259,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-elasticsearch/v7 v7.14.0 h1:extp3jos/rwJn3J+lgbaGlwAgs0TVsIHme00GyNAyX4= github.com/elastic/go-elasticsearch/v7 v7.14.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-sysinfo v1.7.0 h1:4vVvcfi255+8+TyQ7TYUTEK3A+G8v5FLE+ZKYL1z1Dg= -github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= -github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= -github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/go-sysinfo v1.15.4 h1:A3zQcunCxik14MgXu39cXFXcIw2sFXZ0zL886eyiv1Q= +github.com/elastic/go-sysinfo v1.15.4/go.mod h1:ZBVXmqS368dOn/jvijV/zHLfakWTYHBZPk3G244lHrU= +github.com/elastic/go-windows v1.0.2 h1:yoLLsAsV5cfg9FLhZ9EXZ2n2sQFKeDYrHenkcivY4vI= +github.com/elastic/go-windows v1.0.2/go.mod h1:bGcDpBzXgYSqM0Gx3DM4+UxFj300SZLixie9u9ixLM8= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= @@ -272,8 +272,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakRE22OTI12k+2LkyY= github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= -github.com/etclabscore/go-openrpc-reflect v0.0.36 h1:kSqNB2U8RVoW4si+4fsv13NGNkRAQ5j78zTUx1qiehk= -github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= +github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= +github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4= @@ -323,8 +323,8 @@ github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 h1:v+zJS5B6pA3ptWZS4t github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7/go.mod h1:V3Y4KbttaCwyg1gwkP7iai8CbQx4mZUGjd3h9GZWLKE= github.com/filecoin-project/go-ds-versioning v0.1.2 h1:to4pTadv3IeV1wvgbCbN6Vqd+fu+7tveXgv/rCEZy6w= github.com/filecoin-project/go-ds-versioning v0.1.2/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= -github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM= -github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= +github.com/filecoin-project/go-f3 v0.8.10 h1:Mm+daAn9EKqTTDY3ICbPTR2i3Opjb4gr6Y7bJ8oCA84= +github.com/filecoin-project/go-f3 v0.8.10/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-fil-commcid v0.3.1 h1:4EfxpHSlvtkOqa9weG2Yt5kxFmPib2xU7Uc9Lbqk7fs= github.com/filecoin-project/go-fil-commcid v0.3.1/go.mod h1:z7Ssf8d7kspF9QRAVHDbZ+43JK4mkhbGH5lyph1TnKY= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= @@ -337,8 +337,8 @@ github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGy github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.1 h1:wl+ZHruCcE9LvwU7blpwWn35XOcRS6+IBg75G7ZzxzY= github.com/filecoin-project/go-hamt-ipld/v3 v3.4.1/go.mod h1:AqjryNfkxffpnqsa5mwnJHlazhVqF6W2nilu+VYKIq8= -github.com/filecoin-project/go-jsonrpc v0.7.1 h1:++oUd7R3aYibLKXS/DsO348Lco+1cJbfCwRiv8awHFQ= -github.com/filecoin-project/go-jsonrpc v0.7.1/go.mod h1:lAUpS8BSVtKaA8+/CFUMA5dokMiSM7n0ehf8bHOFdpE= +github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= +github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs= github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8= @@ -359,8 +359,8 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus= github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= -github.com/filecoin-project/lotus v1.33.1 h1:eMZ1DBwnJm7BT++psUEDi0XxNRzpUK8qGiNYBCbBYZE= -github.com/filecoin-project/lotus v1.33.1/go.mod h1:70y4vylPpIetPXr+GMF88ssSPG9X5YCh1Ev1PkxEQzc= +github.com/filecoin-project/lotus v1.34.0-rc2 h1:ZeqFyPetdB/omkOeeZKPmDCzP1L4dgYsQLn8dZmEyKM= +github.com/filecoin-project/lotus v1.34.0-rc2/go.mod h1:L23R+rrmgSbEYzc75+f5R0mirVs738N4B2cCl9agbqA= github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM= github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -402,8 +402,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.2.0 h1:vSyEgKwraXPSOkvCk7IwOSyX+Pv3V2cV9CikJMXg4U4= github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU= -github.com/georgysavva/scany/v2 v2.1.3 h1:Zd4zm/ej79Den7tBSU2kaTDPAH64suq4qlQdhiBeGds= -github.com/georgysavva/scany/v2 v2.1.3/go.mod h1:fqp9yHZzM/PFVa3/rYEC57VmDx+KDch0LoqrJzkvtos= +github.com/georgysavva/scany/v2 v2.1.4 h1:nrzHEJ4oQVRoiKmocRqA1IyGOmM/GQOEsg9UjMR5Ip4= +github.com/georgysavva/scany/v2 v2.1.4/go.mod h1:fqp9yHZzM/PFVa3/rYEC57VmDx+KDch0LoqrJzkvtos= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -770,8 +770,8 @@ github.com/ipld/frisbii v0.6.1/go.mod h1:5alsRVbOyUbZ2In70AdJ4VOLh13LkmAMUomotJa github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= -github.com/ipld/go-car/v2 v2.14.3 h1:1Mhl82/ny8MVP+w1M4LXbj4j99oK3gnuZG2GmG1IhC8= -github.com/ipld/go-car/v2 v2.14.3/go.mod h1:/vpSvPngOX8UnvmdFJ3o/mDgXa9LuyXsn7wxOzHDYQE= +github.com/ipld/go-car/v2 v2.15.0 h1:RxtZcGXFx72zFESl+UUsCNQV2YMcy3gEMYx9M3uio24= +github.com/ipld/go-car/v2 v2.15.0/go.mod h1:ovlq/n3xlVJDmoiN3Kd/Z7kIzQbdTIFSwltfOP+qIgk= github.com/ipld/go-codec-dagpb v1.7.0 h1:hpuvQjCSVSLnTnHXn+QAMR0mLmb1gA6wl10LExo2Ts0= github.com/ipld/go-codec-dagpb v1.7.0/go.mod h1:rD3Zg+zub9ZnxcLwfol/OTQRVjaLzXypgy4UqHQvilM= github.com/ipld/go-fixtureplate v0.0.3 h1:Qb/rBBnYP8IiK+VLq89y2NPZ3iQeQpAi9YK3oSleVGs= @@ -828,8 +828,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -1002,8 +1000,8 @@ github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -1062,8 +1060,8 @@ github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.16.0 h1:oGWEVKioVQcdIOBlYM8BH1rZDWOGJSqr9/BKl6zQ4qc= -github.com/multiformats/go-multiaddr v0.16.0/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= +github.com/multiformats/go-multiaddr v0.16.1 h1:fgJ0Pitow+wWXzN9do+1b8Pyjmo8m5WhGfzpL82MpCw= +github.com/multiformats/go-multiaddr v0.16.1/go.mod h1:JSVUmXDjsVFiW7RjIFMP7+Ev+h1DTbiJgVeTV/tcmP0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= @@ -1231,7 +1229,6 @@ github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQP github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -1241,8 +1238,8 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag= -github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= +github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU= +github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= @@ -1456,8 +1453,8 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= -github.com/zondax/ledger-filecoin-go v1.0.1 h1:Ox5u//5PEZhNOAaNtfKqZkF0XvrIVXK5O7DfykJMqu4= -github.com/zondax/ledger-filecoin-go v1.0.1/go.mod h1:L9dAY5dqCISd/tpU912Hzct72Gr+D4o2psRja9udH1Y= +github.com/zondax/ledger-filecoin-go v1.1.0 h1:sDHycC/e4wbniEb8i95pctR3vr0kg2tOt/K9Opvcu2Y= +github.com/zondax/ledger-filecoin-go v1.1.0/go.mod h1:IkszJ5Fp0vAACKBgZ7TbyBnSltFOkWGffvGa8ojgGMA= github.com/zondax/ledger-go v1.0.0 h1:BvNoksIyRqyQTW78rIZP9A44WwAminKiomQa7jXp9EI= github.com/zondax/ledger-go v1.0.0/go.mod h1:HpgkgFh3Jkwi9iYLDATdyRxc8CxqxcywsFj6QerWzvo= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= @@ -1503,10 +1500,10 @@ go.opentelemetry.io/otel/exporters/prometheus v0.50.0 h1:2Ewsda6hejmbhGFyUvWZjUT go.opentelemetry.io/otel/exporters/prometheus v0.50.0/go.mod h1:pMm5PkUo5YwbLiuEf7t2xg4wbP0/eSJrMxIMxKosynY= go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= @@ -1569,8 +1566,8 @@ golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1608,8 +1605,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1665,8 +1662,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1691,8 +1688,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1724,7 +1721,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1776,8 +1772,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1787,8 +1783,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1801,8 +1797,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1864,8 +1860,8 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1934,10 +1930,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20250212204824-5a70512c5d8b h1:i+d0RZa8Hs2L/MuaOQYI+krthcxdEbEM2N+Tf3kJ4zk= -google.golang.org/genproto/googleapis/api v0.0.0-20250212204824-5a70512c5d8b/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b h1:FQtJ1MxbXoIIrZHZ33M+w5+dAP9o86rgpjoKr/ZmT7k= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= +google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 h1:hE3bRWtU6uceqlh4fhrSnUyjKHMKB9KrTLLG+bc0ddM= +google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463/go.mod h1:U90ffi8eUL9MwPcrJylN5+Mk2v3vuPDptd5yyNUiRR8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1954,8 +1950,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/lib/ffi/sdr_funcs.go b/lib/ffi/sdr_funcs.go index 0c792b309..38c84b391 100644 --- a/lib/ffi/sdr_funcs.go +++ b/lib/ffi/sdr_funcs.go @@ -648,14 +648,9 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector defer releaseSector() - ssize, err := sector.ProofType.SectorSize() - if err != nil { - return xerrors.Errorf("getting sector size: %w", err) - } - // If this is a synthetic proof sector then unsealed should already exist otherwise generate it if abi.Synthetic[sector.ProofType] { - if err := ffi.ClearSyntheticProofs(uint64(ssize), sectorPaths.Cache); err != nil { + if err := ffi.ClearSyntheticProofs(sectorPaths.Cache); err != nil { return xerrors.Errorf("Unable to delete Synth cache: %w", err) } } else { @@ -664,7 +659,7 @@ func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.Sector } } - if err := ffi.ClearCache(uint64(ssize), sectorPaths.Cache); err != nil { + if err := ffi.ClearCache(sectorPaths.Cache); err != nil { return xerrors.Errorf("clearing cache: %w", err) } @@ -799,11 +794,6 @@ func (sb *SealCalls) SyntheticProofs(ctx context.Context, task *harmonytask.Task } defer releaseSector() - ssize, err := sector.ProofType.SectorSize() - if err != nil { - return err - } - err = ffi.GenerateSynthProofs(sector.ProofType, sealed, unsealed, fspaths.Cache, fspaths.Sealed, sector.ID.Number, sector.ID.Miner, randomness, pieces) if err != nil { return xerrors.Errorf("generating synthetic proof: %w", err) @@ -824,7 +814,7 @@ func (sb *SealCalls) SyntheticProofs(ctx context.Context, task *harmonytask.Task return xerrors.Errorf("generating unsealed sector: %w", err) } - if err = ffi.ClearCache(uint64(ssize), fspaths.Cache); err != nil { + if err = ffi.ClearCache(fspaths.Cache); err != nil { return xerrors.Errorf("failed to clear cache for synthetic proof of sector %d of miner %d", sector.ID.Miner, sector.ID.Number) } diff --git a/lib/ffi/snap_funcs.go b/lib/ffi/snap_funcs.go index acdaedd49..46eacb46f 100644 --- a/lib/ffi/snap_funcs.go +++ b/lib/ffi/snap_funcs.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/curio/lib/ffiselect" paths2 "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/proof" - storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/tarutil" "github.com/filecoin-project/lotus/storage/sealer/fr32" @@ -242,17 +242,12 @@ func (sb *SealCalls) EncodeUpdate( return cid.Undef, cid.Undef, xerrors.Errorf("write vanilla proofs: %w", err) } - ssize, err := sector.ProofType.SectorSize() - if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("getting sector size: %w", err) - } - // cleanup if err := cleanupStagedFiles(); err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("cleanup staged files: %w", err) } - if err := ffi.ClearCache(uint64(ssize), paths.UpdateCache); err != nil { + if err := ffi.ClearCache(paths.UpdateCache); err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("clear cache: %w", err) } diff --git a/market/mk20/client/client.go b/market/mk20/client/client.go index 1c2ce7b84..414774362 100644 --- a/market/mk20/client/client.go +++ b/market/mk20/client/client.go @@ -6,15 +6,17 @@ import ( "crypto/rand" "io" - lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/wallet" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/oklog/ulid" "golang.org/x/xerrors" - "github.com/filecoin-project/curio/market/mk20" "github.com/filecoin-project/go-address" + + "github.com/filecoin-project/curio/market/mk20" + + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/wallet" ) var log = logging.Logger("mk20-client") diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go index af5a42171..e36a3582f 100644 --- a/market/mk20/types_test.go +++ b/market/mk20/types_test.go @@ -282,7 +282,7 @@ func TestDeal_Products_OmitEmptyInnerFields(t *testing.T) { } func TestPartialUnmarshal(t *testing.T) { - iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdpV1\":{\"createDataSet\":true,\"addPiece\":true,\"recordKeeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" + iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdp_v1\":{\"create_data_set\":true,\"add_piece\":true,\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" var deal Deal if err := json.Unmarshal([]byte(iString), &deal); err != nil { t.Fatal(err) diff --git a/tasks/pdp/task_commp.go b/tasks/pdp/task_commp.go index 20cca9f00..aa2c244c3 100644 --- a/tasks/pdp/task_commp.go +++ b/tasks/pdp/task_commp.go @@ -8,15 +8,13 @@ import ( "strconv" "time" - "github.com/filecoin-project/curio/lib/passcall" - "github.com/filecoin-project/curio/market/mk20" - commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/ipfs/go-cid" "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-commp-utils/writer" commcid "github.com/filecoin-project/go-fil-commcid" + commp "github.com/filecoin-project/go-fil-commp-hashhash" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" @@ -25,7 +23,9 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/market/mk20" ) type PDPCommpTask struct { From 521718283cbbe73927b7e03817bee924a613bd8f Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Tue, 9 Sep 2025 23:02:04 +0400 Subject: [PATCH 43/55] fix status, add status command --- cmd/sptool/toolbox_deal_client.go | 62 +++++++++- cmd/sptool/toolbox_deal_tools.go | 4 +- market/mk20/mk20_utils.go | 197 +++++++++++++++++++++++------- market/mk20/utils.go | 4 +- 4 files changed, 219 insertions(+), 48 deletions(-) diff --git a/cmd/sptool/toolbox_deal_client.go b/cmd/sptool/toolbox_deal_client.go index 915c895e9..3b14b52d5 100644 --- a/cmd/sptool/toolbox_deal_client.go +++ b/cmd/sptool/toolbox_deal_client.go @@ -704,7 +704,7 @@ var initCmd = &cli.Command{ return err } - _ = os.Mkdir(sdir, 0755) //nolint:errcheck + _ = os.Mkdir(sdir, 0755) n, err := Setup(cctx.String(mk12_client_repo.Name)) if err != nil { @@ -1592,6 +1592,7 @@ var mk20Clientcmd = &cli.Command{ mk20ClientMakeAggregateCmd, mk20ClientUploadCmd, mk20ClientChunkUploadCmd, + mk20PDPDealStatusCmd, }, } @@ -2466,3 +2467,62 @@ var mk20ClientUploadCmd = &cli.Command{ return nil }, } + +var mk20PDPDealStatusCmd = &cli.Command{ + Name: "deal-status", + Usage: "Get status of a Mk20 deal", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "provider", + Usage: "PDP providers's URL", + Required: true, + }, + &cli.StringFlag{ + Name: "id", + Usage: "deal id", + Required: true, + }, + &cli.StringFlag{ + Name: "wallet", + Usage: "wallet address to be used to initiate the deal", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + n, err := Setup(cctx.String(mk12_client_repo.Name)) + if err != nil { + return err + } + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return err + } + + maddr, err := url.Parse(cctx.String("provider")) + if err != nil { + return err + } + + pclient := client.NewClient(maddr.String(), walletAddr, n.Wallet) + + status, err := pclient.DealStatus(ctx, cctx.String("id")) + if err != nil { + return xerrors.Errorf("getting deal status: %w", err) + } + + if status.PDPV1 != nil { + fmt.Println("PDP Status:") + fmt.Println("State:", status.PDPV1.State) + fmt.Println("Error:", status.PDPV1.ErrorMsg) + } + + if status.DDOV1 != nil { + fmt.Println("PDP Status:") + fmt.Println("State:", status.DDOV1.State) + fmt.Println("Error:", status.DDOV1.ErrorMsg) + } + + return nil + }, +} diff --git a/cmd/sptool/toolbox_deal_tools.go b/cmd/sptool/toolbox_deal_tools.go index 6fb83e41b..b8dcddaf6 100644 --- a/cmd/sptool/toolbox_deal_tools.go +++ b/cmd/sptool/toolbox_deal_tools.go @@ -224,7 +224,9 @@ var commpCmd = &cli.Command{ if err != nil { return err } - defer rdr.Close() //nolint:errcheck + defer func() { + _ = rdr.Close() + }() w := &writer.Writer{} _, err = io.CopyBuffer(w, rdr, make([]byte, writer.CommPBuf)) diff --git a/market/mk20/mk20_utils.go b/market/mk20/mk20_utils.go index 425b2ba1b..bc708e229 100644 --- a/market/mk20/mk20_utils.go +++ b/market/mk20/mk20_utils.go @@ -20,8 +20,6 @@ import ( // @Return *DealProductStatusResponse func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { - // Check if we ever accepted this deal - var pdp_complete, ddo_complete sql.NullBool var pdp_error, ddo_error sql.NullString @@ -43,51 +41,179 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { HTTPCode: http.StatusInternalServerError, } } - // Handle corner case if now product rows - if !pdp_complete.Valid && !ddo_complete.Valid { + + deal, err := DealFromDB(ctx, m.DB, id) + if err != nil { + log.Errorw("failed to get deal from db", "deal", id, "error", err) return &DealStatus{ - HTTPCode: http.StatusNotFound, + HTTPCode: http.StatusInternalServerError, } } - ret := &DealStatus{ - HTTPCode: http.StatusOK, - } + isPDP := deal.Products.PDPV1 != nil + isDDO := deal.Products.DDOV1 != nil - if pdp_complete.Valid { - if pdp_complete.Bool && !pdp_error.Valid { + // If only PDP is defined + if isPDP && !isDDO { + ret := &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealProductStatusResponse{ + PDPV1: &DealStatusResponse{ + State: DealStateAccepted, + }, + }, + } + if pdp_complete.Bool { ret.Response.PDPV1.State = DealStateComplete } - if pdp_complete.Bool && pdp_error.Valid { + if pdp_error.Valid && pdp_error.String != "" { ret.Response.PDPV1.State = DealStateFailed ret.Response.PDPV1.ErrorMsg = pdp_error.String } + + if !pdp_complete.Bool { + pdp := deal.Products.PDPV1 + if pdp.AddPiece { + if deal.Data != nil { + // Check if deal is uploaded + var yes bool + err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_upload_waiting WHERE id = $1)`, id.String()).Scan(&yes) + if err != nil { + log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + if yes { + ret.Response.PDPV1.State = DealStateAwaitingUpload + } else { + ret.Response.PDPV1.State = DealStateProcessing + } + } else { + ret.Response.PDPV1.State = DealStateAccepted + } + } + + if pdp.CreateDataSet || pdp.DeleteDataSet || pdp.DeletePiece { + ret.Response.PDPV1.State = DealStateProcessing + } + } + + return ret } - if ddo_complete.Valid { - if ddo_complete.Bool && !ddo_error.Valid { + // If only DDO is defined + if isDDO && !isPDP { + ret := &DealStatus{ + HTTPCode: http.StatusOK, + Response: &DealProductStatusResponse{ + DDOV1: &DealStatusResponse{ + State: DealStateAccepted, + }, + }, + } + if ddo_complete.Bool { ret.Response.DDOV1.State = DealStateComplete } - if ddo_complete.Bool && ddo_error.Valid { + if ddo_error.Valid && ddo_error.String != "" { ret.Response.DDOV1.State = DealStateFailed ret.Response.DDOV1.ErrorMsg = ddo_error.String } + + if !ddo_complete.Bool { + state, err := m.getDDOStatus(ctx, id) + if err != nil { + log.Errorw("failed to get DDO status", "deal", id.String(), "error", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + ret.Response.DDOV1.State = state + } + + return ret } - if ret.Response.DDOV1.State == DealStateComplete && ret.Response.PDPV1.State == DealStateComplete { + // If both PDP and DDO are defined + if isPDP && isDDO { + ret := &DealStatus{ + HTTPCode: http.StatusOK, + } + + if pdp_complete.Bool { + ret.Response.PDPV1.State = DealStateComplete + } + + if pdp_error.Valid { + ret.Response.PDPV1.State = DealStateFailed + ret.Response.PDPV1.ErrorMsg = pdp_error.String + } + + if ddo_complete.Bool { + ret.Response.DDOV1.State = DealStateComplete + } + + if ddo_error.Valid && ddo_error.String != "" { + ret.Response.DDOV1.State = DealStateFailed + ret.Response.DDOV1.ErrorMsg = ddo_error.String + } + + if !pdp_complete.Bool { + pdp := deal.Products.PDPV1 + if pdp.AddPiece { + if deal.Data != nil { + // Check if deal is uploaded + var yes bool + err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_upload_waiting WHERE id = $1)`, id.String()).Scan(&yes) + if err != nil { + log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + if yes { + ret.Response.PDPV1.State = DealStateAwaitingUpload + } else { + ret.Response.PDPV1.State = DealStateProcessing + } + } else { + ret.Response.PDPV1.State = DealStateAccepted + } + } + + if pdp.CreateDataSet || pdp.DeleteDataSet || pdp.DeletePiece { + ret.Response.PDPV1.State = DealStateProcessing + } + } + + if !ddo_complete.Bool { + state, err := m.getDDOStatus(ctx, id) + if err != nil { + log.Errorw("failed to get DDO status", "deal", id.String(), "error", err) + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + } + ret.Response.DDOV1.State = state + } + return ret } + return &DealStatus{ + HTTPCode: http.StatusInternalServerError, + } + +} + +func (m *MK20) getDDOStatus(ctx context.Context, id ulid.ULID) (DealState, error) { var waitingForPipeline bool - err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_pipeline_waiting WHERE id = $1)`, id.String()).Scan(&waitingForPipeline) + err := m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM market_mk20_pipeline_waiting WHERE id = $1)`, id.String()).Scan(&waitingForPipeline) if err != nil { - log.Errorw("failed to query the db for deal status", "deal", id.String(), "err", err) - return &DealStatus{ - HTTPCode: http.StatusInternalServerError, - } + return DealStateAccepted, err } if waitingForPipeline { - ret.Response.DDOV1.State = DealStateAccepted + return DealStateAccepted, nil } var pdeals []struct { @@ -106,45 +232,28 @@ func (m *MK20) DealStatus(ctx context.Context, id ulid.ULID) *DealStatus { id = $1`, id.String()) if err != nil { - log.Errorw("failed to query the db for deal pipeline status", "deal", id.String(), "err", err) - return &DealStatus{ - HTTPCode: http.StatusInternalServerError, - } + return DealStateAccepted, err } if len(pdeals) > 1 { - ret.Response.DDOV1.State = DealStateProcessing + return DealStateProcessing, nil } // If deal is still in pipeline if len(pdeals) == 1 { pdeal := pdeals[0] if pdeal.Sector == nil { - ret.Response.DDOV1.State = DealStateProcessing + return DealStateProcessing, nil } if !pdeal.Sealed { - ret.Response.DDOV1.State = DealStateSealing + return DealStateSealing, nil } if !pdeal.Indexed { - ret.Response.DDOV1.State = DealStateIndexing + return DealStateIndexing, nil } } - var pdpPipeline bool - err = m.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pdp_pipeline WHERE id = $1)`, id.String()).Scan(&pdpPipeline) - if err != nil { - log.Errorw("failed to query the db for PDP deal status", "deal", id.String(), "err", err) - return &DealStatus{ - HTTPCode: http.StatusInternalServerError, - } - } - if waitingForPipeline { - ret.Response.PDPV1.State = DealStateProcessing - } else { - ret.Response.PDPV1.State = DealStateAccepted - } - - return ret + return DealStateComplete, nil } // Supported retrieves and returns maps of product names and data source names with their enabled status, or an error if the query fails. diff --git a/market/mk20/utils.go b/market/mk20/utils.go index e475f0122..c686ac053 100644 --- a/market/mk20/utils.go +++ b/market/mk20/utils.go @@ -681,10 +681,10 @@ type DealStatusResponse struct { type DealProductStatusResponse struct { // DDOV1 holds the DealStatusResponse for product "ddo_v1". - DDOV1 DealStatusResponse `json:"ddo_v1"` + DDOV1 *DealStatusResponse `json:"ddo_v1,omitempty"` // PDPV1 represents the DealStatusResponse for the product pdp_v1. - PDPV1 DealStatusResponse `json:"pdp_v1"` + PDPV1 *DealStatusResponse `json:"pdp_v1,omitempty"` } // DealStatus represents the status of a deal, including the HTTP code and an optional response detailing the deal's state and error message. From 5d90eea0462604d7e396328f8709f83f6cbfb4c8 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Tue, 9 Sep 2025 15:47:13 -0500 Subject: [PATCH 44/55] some progress --- market/mk20/tsclient/examples/basic-usage.ts | 128 -------------- .../mk20/tsclient/examples/pdpv1-workflow.ts | 89 ---------- .../mk20/tsclient/examples/streaming-pdp.ts | 36 ---- .../tsclient/examples/unpkg-end-to-end.ts | 19 +-- .../mk20/tsclient/examples/upload-methods.ts | 161 ------------------ market/mk20/tsclient/src/client.ts | 118 ++++++++++--- market/mk20/tsclient/src/streaming.ts | 6 +- pdp/contract/addresses.go | 2 +- 8 files changed, 99 insertions(+), 460 deletions(-) delete mode 100644 market/mk20/tsclient/examples/basic-usage.ts delete mode 100644 market/mk20/tsclient/examples/pdpv1-workflow.ts delete mode 100644 market/mk20/tsclient/examples/streaming-pdp.ts delete mode 100644 market/mk20/tsclient/examples/upload-methods.ts diff --git a/market/mk20/tsclient/examples/basic-usage.ts b/market/mk20/tsclient/examples/basic-usage.ts deleted file mode 100644 index 4949d9052..000000000 --- a/market/mk20/tsclient/examples/basic-usage.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { Client, MarketClientConfig, Deal, DataSource, Products, DDOV1, RetrievalV1 } from '../src'; - -// Example configuration -const config: MarketClientConfig = { - serverUrl: 'http://localhost:8080', - // Optional: Add custom headers - headers: { - 'Authorization': 'Bearer your-token-here' - } -}; - -// Create client instance -const client = new Client(config); - -async function exampleUsage() { - try { - // Get supported contracts - console.log('Getting supported contracts...'); - const contracts = await client.getContracts(); - console.log('Contracts:', contracts); - - // Get supported products - console.log('\nGetting supported products...'); - const products = await client.getProducts(); - console.log('Products:', products); - - // Get supported data sources - console.log('\nGetting supported data sources...'); - const sources = await client.getSources(); - console.log('Sources:', sources); - - // Example: Submit a deal - console.log('\nSubmitting a deal...'); - const deal: Deal = { - identifier: '01H0EXAMPLEULIDIDENTIFIER00000000', // Example ULID string - client: 'f1abcdefghijklmnopqrstuvwxyz123456789', - data: { - pieceCid: 'bafybeihq6mbsd757cdm4sn6z5r7w6tdvkrb3q9iu3pjr7q3ip24c65qh2i', - format: { - raw: {} - }, - sourceHttpput: { - raw_size: 1024 * 1024 // 1MB - } as unknown as object - } as DataSource, - products: { - ddoV1: { - duration: 518400, // Typical lifespan value (epochs) - provider: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, - contractAddress: '0x1234567890123456789012345678901234567890', - contractVerifyMethod: 'verifyDeal', - contractVerifyMethodParams: '', - pieceManager: { address: 'f1abcdefghijklmnopqrstuvwxyz123456789' }, - notificationAddress: 'f1abcdefghijklmnopqrstuvwxyz123456789', - notificationPayload: '' - } as DDOV1, - retrievalV1: { - announcePayload: true, // Announce payload to IPNI - announcePiece: true, // Announce piece information to IPNI - indexing: true // Index for CID-based retrieval - } as RetrievalV1 - } as Products - }; - - const result = await client.submitDeal(deal); - console.log('Deal submitted:', result); - - // Get deal status - if (result && result === 200) { // DealCode.Ok - console.log('\nGetting deal status...'); - const status = await client.getStatus('example-deal-id'); - console.log('Deal status:', status); - } - - } catch (error) { - console.error('Error:', error); - } -} - -// Example: Upload data for a deal -async function uploadDataExample(dealId: string, data: number[]) { - try { - console.log(`Uploading data for deal ${dealId}...`); - await client.uploadData(dealId, data); - console.log('Data uploaded successfully'); - } catch (error) { - console.error('Upload failed:', error); - } -} - -// Example: Demonstrate piece ID calculation for individual blobs -async function pieceIdCalculationExample() { - try { - console.log('šŸ” Piece ID Calculation Example'); - console.log('Calculating piece IDs for individual blobs...\n'); - - // Create mock blobs with different content - const mockBlobs = [ - new Blob(['file1 content'], { type: 'text/plain' }), - new Blob(['file2 content'], { type: 'text/plain' }), - new Blob(['file3 content'], { type: 'text/plain' }) - ]; - - // Use the convenience wrapper to see piece IDs - const result = await client.submitPDPv1DealWithUpload({ - blobs: mockBlobs, - client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', - recordKeeper: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', - contractAddress: '0x1234567890123456789012345678901234567890' - }); - - console.log('šŸ“‹ Deal and Upload Results:'); - console.log('UUID:', result.uuid); - console.log('Total Size:', result.totalSize, 'bytes'); - console.log('Deal ID:', result.dealId); - console.log('Piece CID:', result.pieceCid); - console.log('Uploaded Chunks:', result.uploadedChunks); - console.log('Uploaded Bytes:', result.uploadedBytes); - - return result; - - } catch (error) { - console.error('āŒ Piece ID calculation example failed:', error); - throw error; - } -} - -export { exampleUsage, uploadDataExample, pieceIdCalculationExample }; diff --git a/market/mk20/tsclient/examples/pdpv1-workflow.ts b/market/mk20/tsclient/examples/pdpv1-workflow.ts deleted file mode 100644 index eff07bcdb..000000000 --- a/market/mk20/tsclient/examples/pdpv1-workflow.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Client, MarketClientConfig } from '../src'; - -const config: MarketClientConfig = { - serverUrl: 'http://localhost:8080', - headers: { 'Authorization': 'Bearer your-token-here' } -}; - -const client = new Client(config); - -// Simple PDPv1 workflow with blob array -async function pdpv1CompleteWorkflowExample() { - try { - console.log('šŸš€ Starting simple PDPv1 workflow...\n'); - - // Create mock blobs (in real usage, these would be actual files) - const mockBlobs = [ - new Blob(['file1 content'], { type: 'text/plain' }), - new Blob(['file2 content'], { type: 'text/plain' }), - new Blob(['file3 content'], { type: 'text/plain' }) - ]; - - // Submit deal and initialize upload using simplified wrapper - const result = await client.submitPDPv1DealWithUpload({ - blobs: mockBlobs, - client: 'f1client123456789abcdefghijklmnopqrstuvwxyz', - recordKeeper: 'f1provider123456789abcdefghijklmnopqrstuvwxyz', - contractAddress: '0x1234567890123456789012345678901234567890' - }); - - console.log('āœ… Deal and upload initialized successfully!'); - console.log('šŸ“‹ Results:', { - uuid: result.uuid, - totalSize: result.totalSize, - dealId: result.dealId, - pieceCid: result.pieceCid, - uploadedChunks: result.uploadedChunks, - uploadedBytes: result.uploadedBytes - }); - - // Upload data in chunks using the actual blobs - console.log('\nšŸ“¤ Starting data upload...'); - const chunkSize = 1024 * 1024; // 1MB chunks - let totalChunks = 0; - let uploadedBytes = 0; - - for (const [fileIndex, blob] of mockBlobs.entries()) { - const fileSize = blob.size; - const fileChunks = Math.ceil(fileSize / chunkSize); - - console.log(`Uploading file ${fileIndex + 1}/${mockBlobs.length} (${fileSize} bytes, ${fileChunks} chunks)...`); - - for (let i = 0; i < fileSize; i += chunkSize) { - const chunk = blob.slice(i, i + chunkSize); - const chunkNum = totalChunks.toString(); - - // Convert blob chunk to array of numbers for upload - const chunkArray = new Uint8Array(await chunk.arrayBuffer()); - const chunkNumbers = Array.from(chunkArray); - - console.log(` Uploading chunk ${chunkNum + 1} (${chunkNumbers.length} bytes)...`); - await client.uploadChunk(result.uploadId, chunkNum, chunkNumbers); - - totalChunks++; - uploadedBytes += chunkNumbers.length; - } - } - - // Finalize upload - console.log('\nšŸ”’ Finalizing upload...'); - const finalizeResult = await client.finalizeChunkedUpload(result.uploadId); - console.log(`āœ… Upload finalized: ${finalizeResult}`); - - // Check status - const uploadStatus = await client.getUploadStatus(result.uploadId); - const dealStatus = await client.getStatus(result.uploadId); - - console.log('šŸ“ˆ Upload Status:', uploadStatus); - console.log('šŸ“ˆ Deal Status:', dealStatus); - console.log('\nšŸŽ‰ Workflow completed successfully!'); - - return result; - - } catch (error) { - console.error('āŒ Workflow failed:', error); - throw error; - } -} - -export { pdpv1CompleteWorkflowExample }; diff --git a/market/mk20/tsclient/examples/streaming-pdp.ts b/market/mk20/tsclient/examples/streaming-pdp.ts deleted file mode 100644 index 5a1a841c1..000000000 --- a/market/mk20/tsclient/examples/streaming-pdp.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { MarketClientConfig } from '../src'; -import { StreamingPDP } from '../src/streaming'; - -// Example usage (now using the strongly-typed StreamingPDP from src) -async function example() { - const config: MarketClientConfig = { - serverUrl: 'http://localhost:8080', - } as MarketClientConfig; - const client = new (require('../src').Client)(config); - const spdp = new StreamingPDP(client, { - client: 'f1client...', - provider: 'f1provider...', - contractAddress: '0x...', - }); - - await spdp.begin(); - - // Simulate streaming writes - spdp.write(new TextEncoder().encode('hello ')); - spdp.write(new TextEncoder().encode('world')); - - const res = await spdp.commit(); - console.log('Streaming PDP completed:', res); -} - -// Only run wenshen executed directly (ts-node/node), not when imported -if (require.main === module) { - example().catch(err => { - console.error('Streaming PDP example failed:', err); - process.exit(1); - }); -} - -export { StreamingPDP }; - - diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end.ts b/market/mk20/tsclient/examples/unpkg-end-to-end.ts index 46682b85d..8467a6509 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end.ts @@ -146,24 +146,7 @@ async function run() { console.error('Submit error body:', text); } } catch (_) {} - // Extra debug: try store-only minimal deal to isolate - try { - const minimal = { - client: clientAddr, - products: { pdpV1: { createDataSet: true, addPiece: true, recordKeeper: recordKeeper } }, - } as any; - const url = config.serverUrl.replace(/\/$/, '') + '/market/mk20/store'; - const r = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: authHeader }, - body: JSON.stringify(minimal), - }); - console.log('Direct /store status:', r.status); - const body = await r.text().catch(() => ''); - console.log('Direct /store body:', body); - } catch (ee) { - console.error('Direct /store failed:', (ee as Error).message); - } + throw e; } const uploadId = prep.id; diff --git a/market/mk20/tsclient/examples/upload-methods.ts b/market/mk20/tsclient/examples/upload-methods.ts deleted file mode 100644 index 304260e69..000000000 --- a/market/mk20/tsclient/examples/upload-methods.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { Client, MarketClientConfig, StartUpload } from '../src'; - -// Example configuration -const config: MarketClientConfig = { - serverUrl: 'https://andyserver.thepianoexpress.com', - headers: { - //'Authorization': 'Bearer your-token-here' - } -}; - -// Create client instance -const client = new Client(config); - -// Example: Chunked upload (suitable for large deals) -async function chunkedUploadExample(dealId: string, largeData: number[], chunkSize: number = 1024 * 1024) { - try { - console.log(`Starting chunked upload for deal ${dealId}...`); - - // Step 1: Initialize the upload - const startUpload: StartUpload = { - rawSize: largeData.length, - chunkSize: chunkSize - }; - - const initResult = await client.initializeChunkedUpload(dealId, startUpload); - console.log('Upload initialized with result:', initResult); - - // Step 2: Upload data in chunks - const chunks: Array<{ chunkNum: string; result: number }> = []; - for (let i = 0; i < largeData.length; i += chunkSize) { - const chunk = largeData.slice(i, i + chunkSize); - const chunkNum = Math.floor(i / chunkSize).toString(); - - console.log(`Uploading chunk ${chunkNum} (${chunk.length} bytes)...`); - const uploadResult = await client.uploadChunk(dealId, chunkNum, chunk); - chunks.push({ chunkNum, result: uploadResult }); - - // Optional: Check upload status periodically - if (chunks.length % 10 === 0) { - const status = await client.getUploadStatus(dealId); - console.log(`Upload status after ${chunks.length} chunks:`, status); - } - } - - console.log(`All ${chunks.length} chunks uploaded successfully`); - - // Step 3: Finalize the upload - console.log('Finalizing upload...'); - const finalizeResult = await client.finalizeChunkedUpload(dealId); - console.log('Upload finalized with result:', finalizeResult); - - console.log('Chunked upload completed successfully'); - - } catch (error) { - console.error('Chunked upload failed:', error); - } -} - -// Example: Monitor upload progress -async function monitoredUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024) { - try { - console.log(`Starting monitored upload for deal ${dealId}...`); - - // Initialize upload - const startUpload: StartUpload = { - rawSize: data.length, - chunkSize: chunkSize - }; - - await client.initializeChunkedUpload(dealId, startUpload); - - // Upload with progress monitoring - const totalChunks = Math.ceil(data.length / chunkSize); - let completedChunks = 0; - - for (let i = 0; i < data.length; i += chunkSize) { - const chunk = data.slice(i, i + chunkSize); - const chunkNum = Math.floor(i / chunkSize).toString(); - - await client.uploadChunk(dealId, chunkNum, chunk); - completedChunks++; - - // Show progress - const progress = ((completedChunks / totalChunks) * 100).toFixed(1); - console.log(`Progress: ${progress}% (${completedChunks}/${totalChunks} chunks)`); - - // Check status every 10 chunks - if (completedChunks % 10 === 0) { - const status = await client.getUploadStatus(dealId); - console.log('Current upload status:', status); - } - } - - // Finalize - const finalizeResult = await client.finalizeChunkedUpload(dealId); - console.log('Upload completed and finalized:', finalizeResult); - - } catch (error) { - console.error('Monitored upload failed:', error); - } -} - -// Example: Error handling and retry logic -async function robustUploadExample(dealId: string, data: number[], chunkSize: number = 1024 * 1024, maxRetries: number = 3) { - try { - console.log(`Starting robust upload for deal ${dealId}...`); - - // Initialize upload - const startUpload: StartUpload = { - rawSize: data.length, - chunkSize: chunkSize - }; - - await client.initializeChunkedUpload(dealId, startUpload); - - // Upload with retry logic - const totalChunks = Math.ceil(data.length / chunkSize); - let completedChunks = 0; - - for (let i = 0; i < data.length; i += chunkSize) { - const chunk = data.slice(i, i + chunkSize); - const chunkNum = Math.floor(i / chunkSize).toString(); - - let retries = 0; - let success = false; - - while (!success && retries < maxRetries) { - try { - await client.uploadChunk(dealId, chunkNum, chunk); - success = true; - completedChunks++; - console.log(`Chunk ${chunkNum} uploaded successfully (${completedChunks}/${totalChunks})`); - } catch (error) { - retries++; - console.warn(`Chunk ${chunkNum} upload failed (attempt ${retries}/${maxRetries}):`, error); - - if (retries >= maxRetries) { - throw new Error(`Failed to upload chunk ${chunkNum} after ${maxRetries} attempts`); - } - - // Wait before retry (exponential backoff) - await new Promise(resolve => setTimeout(resolve, Math.pow(2, retries) * 1000)); - } - } - } - - // Finalize - const finalizeResult = await client.finalizeChunkedUpload(dealId); - console.log('Robust upload completed successfully:', finalizeResult); - - } catch (error) { - console.error('Robust upload failed:', error); - throw error; - } -} - -export { - chunkedUploadExample, - monitoredUploadExample, - robustUploadExample -}; diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index 2ce4eaa90..405618653 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -278,6 +278,24 @@ export class PieceCidUtils { export class MarketClient { private api: DefaultApi; + + /** + * Try to extract a human-friendly error string from an HTTP Response. + */ + private async formatHttpError(prefix: string, resp: Response): Promise { + const status = resp.status; + const statusText = resp.statusText || ''; + const h = resp.headers; + const reasonHeader = h.get('Reason') || h.get('reason') || h.get('X-Reason') || h.get('x-reason') || h.get('X-Error') || h.get('x-error') || ''; + let body = ''; + try { + // clone() to avoid consuming the body in case other handlers need it + body = await resp.clone().text(); + } catch {} + const details = [reasonHeader?.trim(), body?.trim()].filter(Boolean).join(' | '); + const statusPart = statusText ? `${status} ${statusText}` : String(status); + return `${prefix} (HTTP ${statusPart})${details ? `: ${details}` : ''}`; + } /** * Create a MarketClient instance. @@ -378,12 +396,24 @@ export class MarketClient { * @param deal - Deal payload matching Mk20Deal schema */ async submitDeal(deal: Mk20Deal): Promise { - try { - const response = await this.api.storePost({ body: deal }); - return response; - } catch (error) { - throw new Error(`Failed to submit deal: ${error}`); + // Use Raw call so we can inspect/handle non-JSON responses gracefully + const apiResp = await this.api.storePostRaw({ body: deal }); + const ct = apiResp.raw.headers.get('content-type') || ''; + if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { + return await apiResp.value(); + } + // Treat non-JSON 2xx as success; return HTTP status code + return apiResp.raw.status; + } catch (error: any) { + // If this is a ResponseError, try to extract HTTP status and body text + const resp = error?.response as Response | undefined; + if (resp) { + const msg = await this.formatHttpError('Failed to submit deal', resp); + throw new Error(msg); + } + // Fallback + throw new Error(`Failed to submit deal: ${error?.message || String(error)}`); } } @@ -458,7 +488,7 @@ export class MarketClient { createDataSet: true, addPiece: false, recordKeeper: recordKeeper, - extraData: '', + extraData: [], deleteDataSet: false, deletePiece: false, } as Mk20PDPV1, @@ -481,13 +511,13 @@ export class MarketClient { data: { pieceCid: pieceCid, format: { raw: {} }, - sourceHttpput: {}, + sourceHttpPut: {}, } as Mk20DataSource, products: { pdpV1: { addPiece: true, recordKeeper: recordKeeper, - extraData: '', + extraData: [], deleteDataSet: false, deletePiece: false, } as Mk20PDPV1, @@ -609,10 +639,20 @@ export class MarketClient { */ async initializeChunkedUpload(id: string, startUpload: Mk20StartUpload): Promise { try { - const result = await this.api.uploadsIdPost({ id, data: startUpload }); - return result; - } catch (error) { - throw new Error(`Failed to initialize chunked upload for deal ${id}: ${error}`); + const apiResp = await this.api.uploadsIdPostRaw({ id, data: startUpload }); + const ct = apiResp.raw.headers.get('content-type') || ''; + if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { + return await apiResp.value(); + } + // Treat non-JSON 2xx as success; return HTTP status code + return apiResp.raw.status; + } catch (error: any) { + const resp = error?.response as Response | undefined; + if (resp) { + const msg = await this.formatHttpError(`Failed to initialize chunked upload for deal ${id}`, resp); + throw new Error(msg); + } + throw new Error(`Failed to initialize chunked upload for deal ${id}: ${error?.message || String(error)}`); } } @@ -630,10 +670,20 @@ export class MarketClient { */ async uploadChunk(id: string, chunkNum: string, data: Array): Promise { try { - const result = await this.api.uploadsIdChunkNumPut({ id, chunkNum, data }); - return result; - } catch (error) { - throw new Error(`Failed to upload chunk ${chunkNum} for deal ${id}: ${error}`); + const apiResp = await this.api.uploadsIdChunkNumPutRaw({ id, chunkNum, data }); + const ct = apiResp.raw.headers.get('content-type') || ''; + if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { + return await apiResp.value(); + } + // Treat non-JSON 2xx as success; return HTTP status code + return apiResp.raw.status; + } catch (error: any) { + const resp = error?.response as Response | undefined; + if (resp) { + const msg = await this.formatHttpError(`Failed to upload chunk ${chunkNum} for deal ${id}`, resp); + throw new Error(msg); + } + throw new Error(`Failed to upload chunk ${chunkNum} for deal ${id}: ${error?.message || String(error)}`); } } @@ -649,10 +699,20 @@ export class MarketClient { */ async finalizeChunkedUpload(id: string, deal?: any): Promise { try { - const result = await this.api.uploadsFinalizeIdPost({ id, body: deal }); - return result; - } catch (error) { - throw new Error(`Failed to finalize chunked upload for deal ${id}: ${error}`); + const apiResp = await this.api.uploadsFinalizeIdPostRaw({ id, body: deal }); + const ct = apiResp.raw.headers.get('content-type') || ''; + if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { + return await apiResp.value(); + } + // Treat non-JSON 2xx as success; return HTTP status code + return apiResp.raw.status; + } catch (error: any) { + const resp = error?.response as Response | undefined; + if (resp) { + const msg = await this.formatHttpError(`Failed to finalize chunked upload for deal ${id}`, resp); + throw new Error(msg); + } + throw new Error(`Failed to finalize chunked upload for deal ${id}: ${error?.message || String(error)}`); } } @@ -663,10 +723,20 @@ export class MarketClient { */ async finalizeSerialUpload(id: string, deal?: Mk20Deal): Promise { try { - const result = await this.api.uploadIdPost({ id, body: deal }); - return result; - } catch (error) { - throw new Error(`Failed to finalize serial upload for deal ${id}: ${error}`); + const apiResp = await this.api.uploadIdPostRaw({ id, body: deal }); + const ct = apiResp.raw.headers.get('content-type') || ''; + if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { + return await apiResp.value(); + } + // Treat non-JSON 2xx as success; return HTTP status code + return apiResp.raw.status; + } catch (error: any) { + const resp = error?.response as Response | undefined; + if (resp) { + const msg = await this.formatHttpError(`Failed to finalize serial upload for deal ${id}`, resp); + throw new Error(msg); + } + throw new Error(`Failed to finalize serial upload for deal ${id}: ${error?.message || String(error)}`); } } diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts index b4fa52db2..b1d79f8ab 100644 --- a/market/mk20/tsclient/src/streaming.ts +++ b/market/mk20/tsclient/src/streaming.ts @@ -140,7 +140,7 @@ export class StreamingPDP { pdpV1: { createDataSet: true, recordKeeper: this.providerAddr, - extraData: '', + extraData: [], pieceIds: undefined, deleteDataSet: false, deletePiece: false, @@ -167,7 +167,7 @@ export class StreamingPDP { pdpV1: { addPiece: true, recordKeeper: this.providerAddr, - extraData: '', + extraData: [], pieceIds: undefined, deleteDataSet: false, deletePiece: false, @@ -242,7 +242,7 @@ export class StreamingPDP { const dataSource: Mk20DataSource = { pieceCid: pieceCid, format: { raw: {} } as Mk20PieceDataFormat, - sourceHttpput: { raw_size: this.totalSize } as unknown as object, + sourceHttpPut: { raw_size: this.totalSize } as unknown as object, }; const finalizedDeal: Deal = { diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index f7a29005f..e39eec488 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -12,7 +12,7 @@ import ( ) const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" -const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" +const PDPCalibnet = "0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6" const PDPTestNet = "0x36BB02036a59147b5062BaF997743923Faef1D9e" type PDPContracts struct { From 0df5b00f4bc5256b0b0c79c365e2ff25c9758592 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 10 Sep 2025 01:07:01 +0400 Subject: [PATCH 45/55] fix swagger types, UI fixes --- Dockerfile | 2 +- documentation/en/curio-cli/sptool.md | 16 ++++ market/mk20/ddo_v1.go | 6 +- market/mk20/http/docs.go | 65 +++++++++------- market/mk20/http/swagger.json | 65 +++++++++------- market/mk20/http/swagger.yaml | 43 +++++------ market/mk20/pdp_v1.go | 4 +- market/mk20/types.go | 6 +- market/mk20/types_test.go | 3 +- pdp/contract/addresses.go | 2 +- tasks/indexing/task_pdp_ipni.go | 2 +- web/api/webrpc/ipni.go | 70 +++++++++++------ web/api/webrpc/market.go | 84 ++++++++++---------- web/static/pages/mk20-deal/deal.mjs | 14 ++-- web/static/pages/pdp/pdp.mjs | 107 -------------------------- web/static/pages/piece/piece-info.mjs | 4 +- 16 files changed, 223 insertions(+), 270 deletions(-) diff --git a/Dockerfile b/Dockerfile index 83ae31d32..2a42de94e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ RUN go install github.com/ipld/go-car/cmd/car@latest \ RUN go install github.com/LexLuthr/piece-server@latest \ && cp $GOPATH/bin/piece-server /usr/local/bin/ -RUN go install github.com/ipni/storetheindex@v0.8.38 \ +RUN go install github.com/ipni/storetheindex@latest \ && cp $GOPATH/bin/storetheindex /usr/local/bin/ RUN go install github.com/ethereum/go-ethereum/cmd/geth@latest \ diff --git a/documentation/en/curio-cli/sptool.md b/documentation/en/curio-cli/sptool.md index 13750bd7e..2cee56b86 100644 --- a/documentation/en/curio-cli/sptool.md +++ b/documentation/en/curio-cli/sptool.md @@ -906,6 +906,7 @@ COMMANDS: aggregate Create a new aggregate from a list of CAR files upload Upload a file to the storage provider chunk-upload Upload a file in chunks to the storage provider + deal-status Get status of a Mk20 deal help, h Shows a list of commands or help for one command OPTIONS: @@ -1032,3 +1033,18 @@ OPTIONS: --wallet value wallet address to be used to initiate the deal --help, -h show help ``` + +#### sptool toolbox mk20-client deal-status +``` +NAME: + sptool toolbox mk20-client deal-status - Get status of a Mk20 deal + +USAGE: + sptool toolbox mk20-client deal-status [command options] + +OPTIONS: + --provider value PDP providers's URL + --id value deal id + --wallet value wallet address to be used to initiate the deal + --help, -h show help +``` diff --git a/market/mk20/ddo_v1.go b/market/mk20/ddo_v1.go index ccf828c94..65b9893ee 100644 --- a/market/mk20/ddo_v1.go +++ b/market/mk20/ddo_v1.go @@ -41,7 +41,7 @@ type DDOV1 struct { Duration abi.ChainEpoch `json:"duration"` // AllocationId represents an allocation identifier for the deal. - AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty" swaggertype:"integer" format:"uint64" example:"1"` + AllocationId *verifreg.AllocationId `json:"allocation_id,omitempty"` // ContractAddress specifies the address of the contract governing the deal ContractAddress string `json:"contract_address"` @@ -50,13 +50,13 @@ type DDOV1 struct { ContractVerifyMethod string `json:"contract_verify_method"` // ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - ContractVerifyMethodParams []byte `json:"contract_verify_method_Params,omitempty" swaggertype:"string" format:"byte"` + ContractVerifyMethodParams []byte `json:"contract_verify_method_Params,omitempty"` // NotificationAddress specifies the address to which notifications will be relayed to when sector is activated NotificationAddress string `json:"notification_address"` // NotificationPayload holds the notification data typically in a serialized byte array format. - NotificationPayload []byte `json:"notification_payload,omitempty" swaggertype:"string" format:"byte"` + NotificationPayload []byte `json:"notification_payload,omitempty"` } func (d *DDOV1) Validate(db *harmonydb.DB, cfg *config.MK20Config) (DealCode, error) { diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index d324adbd0..79bfd4a2f 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -845,6 +845,18 @@ const docTemplate = `{ "address.Address": { "type": "object" }, + "cid.Cid": { + "type": "object" + }, + "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { + "type": "integer", + "enum": [ + 0 + ], + "x-enum-varnames": [ + "NoAllocationID" + ] + }, "http.Header": { "type": "object", "additionalProperties": { @@ -870,9 +882,11 @@ const docTemplate = `{ "properties": { "allocation_id": { "description": "AllocationId represents an allocation identifier for the deal.", - "type": "integer", - "format": "uint64", - "example": 1 + "allOf": [ + { + "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" + } + ] }, "contract_address": { "description": "ContractAddress specifies the address of the contract governing the deal", @@ -884,8 +898,10 @@ const docTemplate = `{ }, "contract_verify_method_Params": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", - "type": "string", - "format": "byte" + "type": "array", + "items": { + "type": "integer" + } }, "duration": { "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", @@ -897,8 +913,10 @@ const docTemplate = `{ }, "notification_payload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", - "type": "string", - "format": "byte" + "type": "array", + "items": { + "type": "integer" + } }, "piece_manager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", @@ -931,9 +949,11 @@ const docTemplate = `{ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "type": "string", - "format": "cid", - "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" + "allOf": [ + { + "$ref": "#/definitions/cid.Cid" + } + ] }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -1015,9 +1035,10 @@ const docTemplate = `{ }, "identifier": { "description": "Identifier represents a unique identifier for the deal in ULID format.", - "type": "string", - "format": "ulid", - "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" + "type": "array", + "items": { + "type": "integer" + } }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", @@ -1172,9 +1193,7 @@ const docTemplate = `{ }, "url": { "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", - "type": "string", - "format": "url", - "example": "http://127.0.0.1:8080/piece/xyz" + "type": "string" } } }, @@ -1191,9 +1210,7 @@ const docTemplate = `{ }, "data_set_id": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", - "type": "integer", - "format": "uint64", - "example": 0 + "type": "integer" }, "delete_data_set": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", @@ -1214,14 +1231,8 @@ const docTemplate = `{ "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { - "type": "integer", - "format": "uint64" - }, - "example": [ - 0, - 1, - 2 - ] + "type": "integer" + } }, "record_keeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index 343c4da92..911da1a63 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -836,6 +836,18 @@ "address.Address": { "type": "object" }, + "cid.Cid": { + "type": "object" + }, + "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { + "type": "integer", + "enum": [ + 0 + ], + "x-enum-varnames": [ + "NoAllocationID" + ] + }, "http.Header": { "type": "object", "additionalProperties": { @@ -861,9 +873,11 @@ "properties": { "allocation_id": { "description": "AllocationId represents an allocation identifier for the deal.", - "type": "integer", - "format": "uint64", - "example": 1 + "allOf": [ + { + "$ref": "#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId" + } + ] }, "contract_address": { "description": "ContractAddress specifies the address of the contract governing the deal", @@ -875,8 +889,10 @@ }, "contract_verify_method_Params": { "description": "ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract", - "type": "string", - "format": "byte" + "type": "array", + "items": { + "type": "integer" + } }, "duration": { "description": "Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID.\nIt must be at least 518400", @@ -888,8 +904,10 @@ }, "notification_payload": { "description": "NotificationPayload holds the notification data typically in a serialized byte array format.", - "type": "string", - "format": "byte" + "type": "array", + "items": { + "type": "integer" + } }, "piece_manager": { "description": "Actor providing AuthorizeMessage (like f1/f3 wallet) able to authorize actions such as managing ACLs", @@ -922,9 +940,11 @@ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "type": "string", - "format": "cid", - "example": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" + "allOf": [ + { + "$ref": "#/definitions/cid.Cid" + } + ] }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -1006,9 +1026,10 @@ }, "identifier": { "description": "Identifier represents a unique identifier for the deal in ULID format.", - "type": "string", - "format": "ulid", - "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" + "type": "array", + "items": { + "type": "integer" + } }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", @@ -1163,9 +1184,7 @@ }, "url": { "description": "URL specifies the HTTP endpoint where the piece data can be fetched.", - "type": "string", - "format": "url", - "example": "http://127.0.0.1:8080/piece/xyz" + "type": "string" } } }, @@ -1182,9 +1201,7 @@ }, "data_set_id": { "description": "DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true.", - "type": "integer", - "format": "uint64", - "example": 0 + "type": "integer" }, "delete_data_set": { "description": "DeleteDataSet indicated that this deal is meant to delete an existing DataSet created by SP for the client.\nDataSetID must be defined.", @@ -1205,14 +1222,8 @@ "description": "PieceIDs is a list of Piece ids in a proof set.", "type": "array", "items": { - "type": "integer", - "format": "uint64" - }, - "example": [ - 0, - 1, - 2 - ] + "type": "integer" + } }, "record_keeper": { "description": "RecordKeeper specifies the record keeper contract address for the new PDP dataset.", diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index 088c6b668..d1bd060b9 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -1,6 +1,14 @@ definitions: address.Address: type: object + cid.Cid: + type: object + github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId: + enum: + - 0 + type: integer + x-enum-varnames: + - NoAllocationID http.Header: additionalProperties: items: @@ -18,10 +26,9 @@ definitions: mk20.DDOV1: properties: allocation_id: + allOf: + - $ref: '#/definitions/github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId' description: AllocationId represents an allocation identifier for the deal. - example: 1 - format: uint64 - type: integer contract_address: description: ContractAddress specifies the address of the contract governing the deal @@ -33,8 +40,9 @@ definitions: contract_verify_method_Params: description: ContractDealIDMethodParams represents encoded parameters for the contract verify method if required by the contract - format: byte - type: string + items: + type: integer + type: array duration: description: |- Duration represents the deal duration in epochs. This value is ignored for the deal with allocationID. @@ -47,8 +55,9 @@ definitions: notification_payload: description: NotificationPayload holds the notification data typically in a serialized byte array format. - format: byte - type: string + items: + type: integer + type: array piece_manager: allOf: - $ref: '#/definitions/address.Address' @@ -67,11 +76,10 @@ definitions: description: Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. piece_cid: + allOf: + - $ref: '#/definitions/cid.Cid' description: PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. - example: bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq - format: cid - type: string source_aggregate: allOf: - $ref: '#/definitions/mk20.DataSourceAggregate' @@ -124,9 +132,9 @@ definitions: identifier: description: Identifier represents a unique identifier for the deal in ULID format. - example: 01ARZ3NDEKTSV4RRFFQ69G5FAV - format: ulid - type: string + items: + type: integer + type: array products: allOf: - $ref: '#/definitions/mk20.Products' @@ -243,8 +251,6 @@ definitions: type: integer url: description: URL specifies the HTTP endpoint where the piece data can be fetched. - example: http://127.0.0.1:8080/piece/xyz - format: url type: string type: object mk20.PDPV1: @@ -260,8 +266,6 @@ definitions: data_set_id: description: DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. - example: 0 - format: uint64 type: integer delete_data_set: description: |- @@ -280,12 +284,7 @@ definitions: type: array piece_ids: description: PieceIDs is a list of Piece ids in a proof set. - example: - - 0 - - 1 - - 2 items: - format: uint64 type: integer type: array record_keeper: diff --git a/market/mk20/pdp_v1.go b/market/mk20/pdp_v1.go index 9a18f1668..242035e23 100644 --- a/market/mk20/pdp_v1.go +++ b/market/mk20/pdp_v1.go @@ -26,13 +26,13 @@ type PDPV1 struct { DeletePiece bool `json:"delete_piece"` // DataSetID is PDP verified contract dataset ID. It must be defined for all deals except when CreateDataSet is true. - DataSetID *uint64 `json:"data_set_id,omitempty" swaggertype:"integer" format:"uint64" example:"0"` + DataSetID *uint64 `json:"data_set_id,omitempty"` // RecordKeeper specifies the record keeper contract address for the new PDP dataset. RecordKeeper string `json:"record_keeper"` // PieceIDs is a list of Piece ids in a proof set. - PieceIDs []uint64 `json:"piece_ids,omitempty" swaggertype:"array,integer" format:"uint64" example:"0,1,2"` + PieceIDs []uint64 `json:"piece_ids,omitempty"` // ExtraData can be used to send additional information to service contract when Verifier action like AddRoot, DeleteRoot etc. are performed. ExtraData []byte `json:"extra_data,omitempty"` diff --git a/market/mk20/types.go b/market/mk20/types.go index 60cfe58d7..74909022f 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -14,7 +14,7 @@ import ( type Deal struct { // Identifier represents a unique identifier for the deal in ULID format. - Identifier ulid.ULID `json:"identifier" swaggertype:"string" format:"ulid" example:"01ARZ3NDEKTSV4RRFFQ69G5FAV"` + Identifier ulid.ULID `json:"identifier"` // Client wallet string for the deal Client string `json:"client"` @@ -41,7 +41,7 @@ type Products struct { type DataSource struct { // PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. - PieceCID cid.Cid `json:"piece_cid" swaggertype:"string" format:"cid" example:"bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"` + PieceCID cid.Cid `json:"piece_cid"` // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` @@ -107,7 +107,7 @@ type DataSourceHTTP struct { type HttpUrl struct { // URL specifies the HTTP endpoint where the piece data can be fetched. - URL string `json:"url" swaggertype:"string" format:"url" example:"http://127.0.0.1:8080/piece/xyz"` + URL string `json:"url"` // HTTPHeaders represents the HTTP headers associated with the URL. Headers http.Header `json:"headers"` diff --git a/market/mk20/types_test.go b/market/mk20/types_test.go index e36a3582f..3d65b9341 100644 --- a/market/mk20/types_test.go +++ b/market/mk20/types_test.go @@ -282,7 +282,8 @@ func TestDeal_Products_OmitEmptyInnerFields(t *testing.T) { } func TestPartialUnmarshal(t *testing.T) { - iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdp_v1\":{\"create_data_set\":true,\"add_piece\":true,\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" + //iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"products\":{\"pdp_v1\":{\"create_data_set\":true,\"add_piece\":true,\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"}}}" + iString := "{\"client\":\"t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai\",\"data\":{\"format\":{\"raw\":{}},\"piece_cid\":\"bafkzcibfxx3meais7dgqlg24253d7s2unmxkczzlrnsoni6zmvjy6vi636nslfyggu3q\",\"source_http_put\":{}},\"identifier\":\"01K4R3EK6QEPASQH8KFPKVBNWR\",\"products\":{\"pdp_v1\":{\"add_piece\":true,\"delete_data_set\":false,\"delete_piece\":false,\"extra_data\":[],\"record_keeper\":\"0x158c8f05A616403589b99BE5d82d756860363A92\"},\"retrieval_v1\":{\"announce_payload\":true,\"announce_piece\":true,\"indexing\":true}}}" var deal Deal if err := json.Unmarshal([]byte(iString), &deal); err != nil { t.Fatal(err) diff --git a/pdp/contract/addresses.go b/pdp/contract/addresses.go index f7a29005f..33124b809 100644 --- a/pdp/contract/addresses.go +++ b/pdp/contract/addresses.go @@ -13,7 +13,7 @@ import ( const PDPMainnet = "0x9C65E8E57C98cCc040A3d825556832EA1e9f4Df6" const PDPCalibnet = "0x4E1e9AB9bf23E9Fe96041E0a2d2f0B99dE27FBb2" -const PDPTestNet = "0x36BB02036a59147b5062BaF997743923Faef1D9e" +const PDPTestNet = "0x4b715ECF38E8526f9EddBC7143e9C44E7D12b3Ca" type PDPContracts struct { PDPVerifier common.Address diff --git a/tasks/indexing/task_pdp_ipni.go b/tasks/indexing/task_pdp_ipni.go index ca30b8978..9f4bcc74c 100644 --- a/tasks/indexing/task_pdp_ipni.go +++ b/tasks/indexing/task_pdp_ipni.go @@ -321,7 +321,7 @@ func (P *PDPIPNITask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (don // Close the channel close(recs) - // Wait till is finished + // Wait till is finished err = eg.Wait() if err != nil { return false, xerrors.Errorf("adding index to chunk (interrupted %t): %w", interrupted, err) diff --git a/web/api/webrpc/ipni.go b/web/api/webrpc/ipni.go index fb6cc6ee4..004a390db 100644 --- a/web/api/webrpc/ipni.go +++ b/web/api/webrpc/ipni.go @@ -19,6 +19,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/lib/commcidv2" + itype "github.com/filecoin-project/curio/market/ipni/types" + "github.com/filecoin-project/curio/market/mk20" ) type IpniAd struct { @@ -94,34 +96,58 @@ func (a *WebRPC) GetAd(ctx context.Context, ad string) (*IpniAd, error) { details := ads[0] - var pi abi.PieceInfo - err = pi.UnmarshalCBOR(bytes.NewReader(details.ContextID)) - if err != nil { - return nil, xerrors.Errorf("failed to unmarshal piece info: %w", err) - } + var pcid, pcid2 cid.Cid + var psize int64 - // Get RawSize from market_piece_deal to calculate PieceCidV2 - var rawSize uint64 - err = a.deps.DB.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1;`, pi.PieceCID, pi.Size).Scan(&rawSize) - if err != nil { - return nil, xerrors.Errorf("failed to get raw size: %w", err) - } + if details.SpID == -1 { + var pi itype.PdpIpniContext + err = pi.Unmarshal(details.ContextID) + if err != nil { + return nil, xerrors.Errorf("failed to unmarshal PDP piece info: %w", err) + } + pcid2 = pi.PieceCID + pInfo, err := mk20.GetPieceInfo(pcid2) + if err != nil { + return nil, xerrors.Errorf("failed to get piece info: %w", err) + } + pcid = pInfo.PieceCIDV1 + psize = int64(pInfo.Size) + } else { + var pi abi.PieceInfo + err = pi.UnmarshalCBOR(bytes.NewReader(details.ContextID)) + if err != nil { + return nil, xerrors.Errorf("failed to unmarshal piece info: %w", err) + } - pcidv2, err := commcidv2.PieceCidV2FromV1(pi.PieceCID, rawSize) - if err != nil { - return nil, xerrors.Errorf("failed to get commp: %w", err) + pcid = pi.PieceCID + psize = int64(pi.Size) + + // Get RawSize from market_piece_deal to calculate PieceCidV2 + var rawSize uint64 + err = a.deps.DB.QueryRow(ctx, `SELECT raw_size FROM market_piece_deal WHERE piece_cid = $1 AND piece_length = $2 LIMIT 1;`, pi.PieceCID, pi.Size).Scan(&rawSize) + if err != nil { + return nil, xerrors.Errorf("failed to get raw size: %w", err) + } + + pcid2, err = commcidv2.PieceCidV2FromV1(pi.PieceCID, rawSize) + if err != nil { + return nil, xerrors.Errorf("failed to get commp: %w", err) + } } - details.PieceCid = pi.PieceCID.String() - size := int64(pi.Size) - details.PieceSize = size - details.PieceCidV2 = pcidv2.String() + details.PieceCid = pcid.String() + details.PieceSize = psize + details.PieceCidV2 = pcid2.String() - maddr, err := address.NewIDAddress(uint64(details.SpID)) - if err != nil { - return nil, err + if details.SpID == -1 { + details.Miner = "PDP" + } else { + maddr, err := address.NewIDAddress(uint64(details.SpID)) + if err != nil { + return nil, err + } + details.Miner = maddr.String() } - details.Miner = maddr.String() if !details.PreviousAd.Valid { details.Previous = "" diff --git a/web/api/webrpc/market.go b/web/api/webrpc/market.go index afbede8e5..0b1e33b9c 100644 --- a/web/api/webrpc/market.go +++ b/web/api/webrpc/market.go @@ -26,6 +26,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/commcidv2" + itype "github.com/filecoin-project/curio/market/ipni/types" "github.com/filecoin-project/curio/market/mk20" lapi "github.com/filecoin-project/lotus/api" @@ -414,47 +415,6 @@ func (a *WebRPC) MK12StorageDealList(ctx context.Context, limit int, offset int) } -// LegacyStorageDealList is deprecated -//func (a *WebRPC) LegacyStorageDealList(ctx context.Context, limit int, offset int) ([]StorageDealList, error) { -// var mk12Summaries []StorageDealList -// -// err := a.deps.DB.Select(ctx, &mk12Summaries, `SELECT -// signed_proposal_cid AS uuid, -// sp_id, -// created_at, -// piece_cid, -// piece_size, -// NULL AS error, -// TRUE AS processed -// FROM market_legacy_deals -// ORDER BY created_at DESC -// LIMIT $1 OFFSET $2;`, limit, offset) -// if err != nil { -// return nil, fmt.Errorf("failed to fetch deal list: %w", err) -// } -// -// for i := range mk12Summaries { -// addr, err := address.NewIDAddress(uint64(mk12Summaries[i].MinerID)) -// if err != nil { -// return nil, err -// } -// mk12Summaries[i].Miner = addr.String() -// pcid, err := cid.Parse(mk12Summaries[i].PieceCidV1) -// if err != nil { -// return nil, xerrors.Errorf("failed to parse v1 piece CID: %w", err) -// } -// commp, err := commcidv2.CommPFromPieceInfo(abi.PieceInfo{ -// PieceCID: pcid, -// Size: abi.PaddedPieceSize(mk12Summaries[i].PieceSize), -// }) -// if err != nil { -// return nil, xerrors.Errorf("failed to get commP from piece info: %w", err) -// } -// mk12Summaries[i].PieceCidV2 = commp.PCidV2().String() -// } -// return mk12Summaries, nil -//} - type WalletBalances struct { Address string `json:"address"` Balance string `json:"balance"` @@ -596,7 +556,7 @@ type PieceInfo struct { CreatedAt time.Time `json:"created_at"` Indexed bool `json:"indexed"` IndexedAT time.Time `json:"indexed_at"` - IPNIAd string `json:"ipni_ad"` + IPNIAd []string `json:"ipni_ads"` Deals []*PieceDeal `json:"deals"` } @@ -670,14 +630,48 @@ func (a *WebRPC) PieceInfo(ctx context.Context, pieceCid string) (*PieceInfo, er return nil, xerrors.Errorf("failed to marshal piece info: %w", err) } + c1 := itype.PdpIpniContext{ + PieceCID: piece, + Payload: true, + } + + c1b, err := c1.Marshal() + if err != nil { + return nil, xerrors.Errorf("failed to marshal PDP piece info: %w", err) + } + fmt.Printf("C1B: %x", c1b) + + c2 := itype.PdpIpniContext{ + PieceCID: piece, + Payload: false, + } + + c2b, err := c2.Marshal() + if err != nil { + return nil, xerrors.Errorf("failed to marshal PDP piece info: %w", err) + } + fmt.Printf("C2B: %x", c2b) + // Get only the latest Ad var ipniAd string err = a.deps.DB.QueryRow(ctx, `SELECT ad_cid FROM ipni WHERE context_id = $1 ORDER BY order_number DESC LIMIT 1`, b.Bytes()).Scan(&ipniAd) - if err != nil && err != pgx.ErrNoRows { - return nil, xerrors.Errorf("failed to get deal ID by piece CID: %w", err) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, xerrors.Errorf("failed to get ad ID by piece CID: %w", err) + } + + var ipniAdPdp string + err = a.deps.DB.QueryRow(ctx, `SELECT ad_cid FROM ipni WHERE context_id = $1 ORDER BY order_number DESC LIMIT 1`, c1b).Scan(&ipniAdPdp) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, xerrors.Errorf("failed to get ad ID by piece CID for PDP: %w", err) + } + + var ipniAdPdp1 string + err = a.deps.DB.QueryRow(ctx, `SELECT ad_cid FROM ipni WHERE context_id = $1 ORDER BY order_number DESC LIMIT 1`, c2b).Scan(&ipniAdPdp1) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, xerrors.Errorf("failed to get ad ID by piece CID for PDP: %w", err) } - ret.IPNIAd = ipniAd + ret.IPNIAd = append(ret.IPNIAd, ipniAd, ipniAdPdp, ipniAdPdp1) return ret, nil } diff --git a/web/static/pages/mk20-deal/deal.mjs b/web/static/pages/mk20-deal/deal.mjs index d173e8289..f0db5906f 100644 --- a/web/static/pages/mk20-deal/deal.mjs +++ b/web/static/pages/mk20-deal/deal.mjs @@ -237,7 +237,7 @@ class DealDetails extends LitElement { - < + ${pdp.data_set_id ? html`` : ``} @@ -249,12 +249,12 @@ class DealDetails extends LitElement { renderRetV1(ret) { if (!ret) return ''; return html` -
    Retrieval v1
    -
    Create DataSet
    Create Piece
    Remove Piece
    Remove Piece
    Remove DataSet
    Record Keeper${pdp.record_keeper}>
    DataSet ID${pdp.data_set_id}
    - - - -
    Indexing${ret.indexing ? 'Yes' : 'No'}
    Announce Piece to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    Announce Payload to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    +
    Retrieval v1
    + + + + +
    Indexing${ret.indexing ? 'Yes' : 'No'}
    Announce Piece to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    Announce Payload to IPNI${ret.announce_payload ? 'Yes' : 'No'}
    `; } } diff --git a/web/static/pages/pdp/pdp.mjs b/web/static/pages/pdp/pdp.mjs index 74a05e853..107a81729 100644 --- a/web/static/pages/pdp/pdp.mjs +++ b/web/static/pages/pdp/pdp.mjs @@ -35,65 +35,10 @@ customElements.define('pdp-info', class PDPElement extends LitElement { } } - toggleAddServiceForm() { - this.showAddServiceForm = !this.showAddServiceForm; - } - toggleAddKeyForm() { this.showAddKeyForm = !this.showAddKeyForm; } - async addService(event) { - event.preventDefault(); - - const nameInput = this.shadowRoot.getElementById('service-name'); - const pubKeyInput = this.shadowRoot.getElementById('service-pubkey'); - - const name = nameInput.value.trim(); - const pubKey = pubKeyInput.value.trim(); - - if (!name || !pubKey) { - alert('Please provide both a name and a public key.'); - return; - } - - try { - // Call the RPC method to add the new PDP service - await RPCCall('AddPDPService', [name, pubKey]); - - // Reset the form - nameInput.value = ''; - pubKeyInput.value = ''; - - // Reload the services - await this.loadServices(); - - // Hide the form - this.showAddServiceForm = false; - } catch (error) { - console.error('Failed to add PDP service:', error); - alert('Failed to add PDP service: ' + (error.message || error)); - } - } - - async removeService(serviceId, serviceName) { - const confirmed = confirm(`Are you sure you want to remove the service "${serviceName}"?`); - if (!confirmed) { - return; - } - - try { - // Call the RPC method to remove the PDP service - await RPCCall('RemovePDPService', [serviceId]); - - // Reload the services - await this.loadServices(); - } catch (error) { - console.error('Failed to remove PDP service:', error); - alert('Failed to remove PDP service: ' + (error.message || error)); - } - } - async addKey(event) { event.preventDefault(); @@ -153,58 +98,6 @@ customElements.define('pdp-info', class PDPElement extends LitElement {
    -

    Services

    - ${this.services.length > 0 ? html` - - - - - - - - - - - ${this.services.map(service => html` - - - - - - - `)} - -
    IDNamePublic KeyAction
    ${service.id}${service.name} - - - -
    - ` : html` -

    No PDP services available.

    - `} - - - - ${this.showAddServiceForm ? html` -
    -
    - - -
    -
    - - -
    - -
    - ` : ''} - -
    -

    Owner Addresses

    ${this.keys.length > 0 ? html` diff --git a/web/static/pages/piece/piece-info.mjs b/web/static/pages/piece/piece-info.mjs index f2f0e9ad4..104335f6e 100644 --- a/web/static/pages/piece/piece-info.mjs +++ b/web/static/pages/piece/piece-info.mjs @@ -104,7 +104,9 @@ customElements.define('piece-info', class PieceInfoElement extends LitElement {
    IPNI AD - ${this.data.ipni_ad ? html`${this.data.ipni_ad}` : 'No Ad Found'} + ${this.data.ipni_ads && this.data.ipni_ads.length > 0 + ? this.data.ipni_ads.map(ad => html`${ad} `) + : 'No Ad Found'}
    From 8857f3f6850fec772fd91c79759038418ec4f281 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 10 Sep 2025 18:02:40 -0500 Subject: [PATCH 46/55] update --- market/mk20/http/docs.go | 23 +- market/mk20/http/swagger.json | 24 +- market/mk20/http/swagger.yaml | 16 +- .../tsclient/examples/unpkg-end-to-end/1.ts | 99 ++++++++ .../tsclient/examples/unpkg-end-to-end/2.ts | 238 ++++++++++++++++++ .../tsclient/examples/unpkg-end-to-end/3.ts | 88 +++++++ .../tsclient/examples/unpkg-end-to-end/4.ts | 110 ++++++++ .../examples/unpkg-end-to-end/README.md | 157 ++++++++++++ .../examples/unpkg-end-to-end/auth.ts | 42 ++++ market/mk20/tsclient/src/auth.ts | 106 ++++++++ market/mk20/tsclient/src/client.ts | 27 +- market/mk20/tsclient/src/streaming.ts | 2 +- market/mk20/tsclient/tsconfig.json | 1 + market/mk20/types.go | 4 +- 14 files changed, 894 insertions(+), 43 deletions(-) create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/1.ts create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/2.ts create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/3.ts create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/4.ts create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/README.md create mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end/auth.ts diff --git a/market/mk20/http/docs.go b/market/mk20/http/docs.go index 79bfd4a2f..12dea1c47 100644 --- a/market/mk20/http/docs.go +++ b/market/mk20/http/docs.go @@ -845,11 +845,9 @@ const docTemplate = `{ "address.Address": { "type": "object" }, - "cid.Cid": { - "type": "object" - }, "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { "type": "integer", + "format": "int64", "enum": [ 0 ], @@ -949,11 +947,13 @@ const docTemplate = `{ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "allOf": [ - { - "$ref": "#/definitions/cid.Cid" - } - ] + "type": "object", + "additionalProperties": { + "type": "string" + }, + "example": { + "/": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" + } }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -1035,10 +1035,9 @@ const docTemplate = `{ }, "identifier": { "description": "Identifier represents a unique identifier for the deal in ULID format.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "ulid", + "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", diff --git a/market/mk20/http/swagger.json b/market/mk20/http/swagger.json index ed1370728..97dcba427 100644 --- a/market/mk20/http/swagger.json +++ b/market/mk20/http/swagger.json @@ -3,7 +3,6 @@ "info": { "description": "Curio market APIs", "title": "Curio Market 2.0 API", - "version": "2.0.0", "contact": {} }, "paths": { @@ -837,11 +836,9 @@ "address.Address": { "type": "object" }, - "cid.Cid": { - "type": "object" - }, "github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId": { "type": "integer", + "format": "int64", "enum": [ 0 ], @@ -941,11 +938,13 @@ }, "piece_cid": { "description": "PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object.", - "allOf": [ - { - "$ref": "#/definitions/cid.Cid" - } - ] + "type": "object", + "additionalProperties": { + "type": "string" + }, + "example": { + "/": "bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq" + } }, "source_aggregate": { "description": "SourceAggregate represents an aggregated source, comprising multiple data sources as pieces.", @@ -1027,10 +1026,9 @@ }, "identifier": { "description": "Identifier represents a unique identifier for the deal in ULID format.", - "type": "array", - "items": { - "type": "integer" - } + "type": "string", + "format": "ulid", + "example": "01ARZ3NDEKTSV4RRFFQ69G5FAV" }, "products": { "description": "Products represents a collection of product-specific information associated with a deal", diff --git a/market/mk20/http/swagger.yaml b/market/mk20/http/swagger.yaml index d1bd060b9..3e01d806f 100644 --- a/market/mk20/http/swagger.yaml +++ b/market/mk20/http/swagger.yaml @@ -1,11 +1,10 @@ definitions: address.Address: type: object - cid.Cid: - type: object github_com_filecoin-project_go-state-types_builtin_v16_verifreg.AllocationId: enum: - 0 + format: int64 type: integer x-enum-varnames: - NoAllocationID @@ -76,10 +75,13 @@ definitions: description: Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. piece_cid: - allOf: - - $ref: '#/definitions/cid.Cid' + additionalProperties: + type: string description: PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. + example: + /: bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq + type: object source_aggregate: allOf: - $ref: '#/definitions/mk20.DataSourceAggregate' @@ -132,9 +134,9 @@ definitions: identifier: description: Identifier represents a unique identifier for the deal in ULID format. - items: - type: integer - type: array + example: 01ARZ3NDEKTSV4RRFFQ69G5FAV + format: ulid + type: string products: allOf: - $ref: '#/definitions/mk20.Products' diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts new file mode 100644 index 000000000..9a424c1cd --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts @@ -0,0 +1,99 @@ +// Step 1: Create Dataset +// This step creates a PDPv1 dataset (first part of startPDPv1DealForUpload) +// Set before running: +// PDP_URL=https://andyserver.thepianoexpress.com +// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet +// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode +// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode +// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 +// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... +// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 + +import { getAuthConfigFromEnv, buildAuthHeader, createClient, sanitizeAuthHeader, runPreflightChecks } from './auth'; +import { Mk20Deal, Mk20Products, Mk20PDPV1, Mk20RetrievalV1 } from '../../generated'; +import { ulid } from 'ulid'; + +async function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function run() { + console.log('šŸš€ Step 1: Creating PDPv1 Dataset'); + console.log(' This is the first step and requires no inputs.'); + + // Get configuration from environment + const config = getAuthConfigFromEnv(); + console.log('Configuration loaded from environment'); + + // Build authentication + const authHeader = await buildAuthHeader(config); + console.log('Auth header (sanitized):', sanitizeAuthHeader(authHeader)); + console.log('Server URL:', config.serverUrl); + + // Create authenticated client + const client = createClient(config, authHeader); + + // Run preflight connectivity checks + console.log('šŸ” Running preflight connectivity checks...'); + await runPreflightChecks(config, authHeader); + + // Create dataset with a fresh identifier (first part of startPDPv1DealForUpload) + console.log('šŸ“ Creating PDPv1 dataset...'); + const datasetId = ulid(); + const createDeal: Mk20Deal = { + identifier: datasetId, + client: config.clientAddr, + products: { + pdpV1: { + createDataSet: true, + addPiece: false, + recordKeeper: config.recordKeeper, + extraData: [], + deleteDataSet: false, + deletePiece: false, + } as Mk20PDPV1, + retrievalV1: { + announcePayload: true, + announcePiece: true, + indexing: true, + } as Mk20RetrievalV1, + } as Mk20Products, + } as Mk20Deal; + + // Submit the dataset creation deal + console.log('šŸ“¤ Submitting dataset creation deal...'); + await client.submitDeal(createDeal); + console.log(` Dataset creation deal submitted with ID: ${datasetId}`); + + // Wait for dataset creation to complete + console.log('ā³ Waiting for dataset creation to complete...'); + for (let i = 0; i < 60; i++) { // up to ~5 minutes with 5s interval + const status = await client.getStatus(datasetId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` Dataset status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') break; + await sleep(5000); + } + + console.log('āœ… Step 1 completed: PDPv1 dataset created'); + console.log(` - Dataset ID: ${datasetId}`); + console.log(` - Client: ${config.clientAddr}`); + console.log(` - Record Keeper: ${config.recordKeeper}`); + console.log(''); + console.log('Next: Run 2.ts to add piece to the dataset'); + + return { + datasetId, + config, + }; +} + +if (require.main === module) { + run().catch(err => { + console.error('Step 1 failed:', err); + process.exit(1); + }); +} + +export { run as startDeal }; diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts new file mode 100644 index 000000000..ebeaa34b6 --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts @@ -0,0 +1,238 @@ +// Step 2: Add Piece and Upload Blobs +// This step adds a piece to the dataset, downloads React.js, and uploads it +// Set before running: +// PDP_URL=https://andyserver.thepianoexpress.com +// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet +// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode +// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode +// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 +// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... +// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 + +import { getAuthConfigFromEnv, buildAuthHeader, createClient, sanitizeAuthHeader, runPreflightChecks } from './auth'; +import { Mk20Deal, Mk20Products, Mk20PDPV1, Mk20RetrievalV1, Mk20DataSource } from '../../generated'; +import { PieceCidUtils } from '../../src'; +import { ulid } from 'ulid'; + +async function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function run(datasetId?: string) { + console.log('šŸ“ Step 2: Adding Piece and Uploading Blobs'); + console.log(' REQUIRED INPUT: Dataset ID from Step 1'); + console.log(' This step downloads React.js, adds piece to dataset, and uploads it'); + + // Get configuration from environment + const config = getAuthConfigFromEnv(); + console.log('Configuration loaded from environment'); + + // Build authentication + const authHeader = await buildAuthHeader(config); + console.log('Auth header (sanitized):', sanitizeAuthHeader(authHeader)); + console.log('Server URL:', config.serverUrl); + + // Create authenticated client + const client = createClient(config, authHeader); + + // Run preflight connectivity checks + console.log('šŸ” Running preflight connectivity checks...'); + await runPreflightChecks(config, authHeader); + + // Use provided datasetId or get from environment + const targetDatasetId = datasetId || process.env.DATASET_ID; + if (!targetDatasetId) { + console.error('āŒ REQUIRED INPUT MISSING: Dataset ID'); + console.error(' This step requires a dataset ID from Step 1.'); + console.error(' Either pass as parameter: run("dataset-id")'); + console.error(' Or set environment variable: export DATASET_ID=your-dataset-id'); + console.error(''); + console.error(' To get the dataset ID, run Step 1 first:'); + console.error(' npx ts-node 1.ts'); + throw new Error('REQUIRED INPUT MISSING: Dataset ID from Step 1'); + } + console.log(` Using dataset ID: ${targetDatasetId}`); + + // Download React.js from unpkg + console.log('šŸ“„ Downloading React.js from unpkg...'); + const url = 'https://unpkg.com/react@18.2.0/umd/react.production.min.js'; + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to download React.js: ${response.status} ${response.statusText}`); + } + + const bytes = new Uint8Array(await response.arrayBuffer()); + const blob = new Blob([Buffer.from(bytes)], { type: 'application/octet-stream' }); + console.log(` Downloaded React.js: ${bytes.length} bytes`); + console.log(` Blob size: ${blob.size} bytes`); + + // Compute piece CID + console.log('šŸ”— Computing piece CID...'); + const pieceCid = await PieceCidUtils.computePieceCidV2([blob]); + console.log(` Piece CID: ${pieceCid}`); + + // Add piece with data under a new identifier (upload id) + console.log('šŸ“ Creating add piece deal...'); + const uploadId = ulid(); + const addPieceDeal: Mk20Deal = { + identifier: uploadId, + client: config.clientAddr, + data: { + pieceCid: { "/": pieceCid } as object, + format: { raw: {} }, + sourceHttpPut: {}, + } as Mk20DataSource, + products: { + pdpV1: { + addPiece: true, + dataSetId: 0, // TODO: get dataset id from response (hardcoded for now) + recordKeeper: config.recordKeeper, + extraData: [], + deleteDataSet: false, + deletePiece: false, + } as Mk20PDPV1, + retrievalV1: { + announcePayload: false, + announcePiece: true, + indexing: false, + } as Mk20RetrievalV1, + } as Mk20Products, + } as Mk20Deal; + + // Submit the add piece deal + console.log('šŸ“¤ Submitting add piece deal...'); + const dealId = await client.submitDeal(addPieceDeal); + console.log(` Add piece deal submitted with ID: ${uploadId}, deal ID: ${dealId}`); + + // Wait for add piece to complete + console.log('ā³ Waiting for add piece to complete...'); + let addPieceComplete = false; + for (let i = 0; i < 12; i++) { // up to 60 seconds with 5s interval + try { + const status = await client.getStatus(uploadId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` Add piece status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') { + addPieceComplete = true; + break; + } + } catch (e) { + console.log(` Status check failed (attempt ${i + 1}): ${(e as Error).message}`); + if (i === 11) { + console.log(' āš ļø Status polling timed out after 60 seconds'); + break; + } + } + await sleep(5000); + } + + if (!addPieceComplete) { + console.log(' ā° Add piece status polling timed out after 60 seconds'); + console.log(' šŸ”— Please check the blockchain for deal status:'); + console.log(` - Upload ID: ${uploadId}`); + console.log(` - Deal ID: ${dealId}`); + console.log(' šŸ“ The deal may still be processing on-chain'); + console.log(' āœ… Proceeding with blob upload (this may still work)'); + } + + // Upload the blobs + console.log('šŸ“¤ Uploading blobs to the deal...'); + try { + const result = await client.uploadBlobs({ + id: uploadId, + blobs: [blob], + deal: addPieceDeal, + chunkSize: 16 * 1024 * 1024 // Use 16MB chunks (server minimum requirement) + }); + console.log(' Blobs uploaded successfully'); + console.log(` - Uploaded chunks: ${result.uploadedChunks}`); + console.log(` - Uploaded bytes: ${result.uploadedBytes}`); + console.log(` - Finalize code: ${result.finalizeCode}`); + } catch (e) { + console.error('Upload error:', (e as Error).message); + try { + const re: any = e as any; + if (re && re.response) { + const status = re.response.status; + const text = await re.response.text().catch(() => ''); + console.error('Upload error status:', status); + console.error('Upload error body:', text); + } + } catch (_) {} + throw e; + } + + // Poll deal status until complete/failed + console.log('ā³ Polling deal status until complete/failed...'); + let finalStatusComplete = false; + for (let i = 0; i < 12; i++) { // up to 60 seconds with 5s interval + try { + const status = await client.getStatus(uploadId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` Status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') { + finalStatusComplete = true; + break; + } + } catch (e) { + console.log(` Status check failed (attempt ${i + 1}): ${(e as Error).message}`); + if (i === 11) { + console.log(' āš ļø Final status polling timed out after 60 seconds'); + break; + } + } + await sleep(5000); + } + + if (!finalStatusComplete) { + console.log(' ā° Final status polling timed out after 60 seconds'); + console.log(' šŸ”— Please check the blockchain for final deal status:'); + console.log(` - Upload ID: ${uploadId}`); + console.log(` - Deal ID: ${dealId}`); + console.log(' šŸ“ The deal may still be processing on-chain'); + console.log(' āœ… Step completed - check chain for final status'); + } + + // Try to get final status, but don't fail if it times out + let finalStatus = null; + try { + finalStatus = await client.getStatus(uploadId); + } catch (e) { + console.log(' āš ļø Could not get final status - check blockchain'); + } + + console.log('āœ… Step 2 completed: Piece added and blobs uploaded'); + console.log(` - Upload ID: ${uploadId}`); + console.log(` - Deal ID: ${dealId}`); + console.log(` - Piece CID: ${pieceCid}`); + console.log(` - Dataset ID: ${targetDatasetId}`); + console.log(` - File size: ${blob.size} bytes`); + if (finalStatus) { + console.log(` - Deal status: ${finalStatus.pdpV1?.status}`); + } else { + console.log(` - Deal status: Check blockchain for final status`); + } + console.log(''); + console.log('Next: Run 3.ts to download and verify the uploaded content'); + + return { + uploadId, + dealId, + pieceCid, + blob, + bytes, + addPieceDeal, + finalStatus: finalStatus?.pdpV1?.status || 'check_blockchain', + }; +} + +if (require.main === module) { + run().catch(err => { + console.error('Step 2 failed:', err); + process.exit(1); + }); +} + +export { run as addPieceAndUpload }; \ No newline at end of file diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/3.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/3.ts new file mode 100644 index 000000000..59f6dec46 --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/3.ts @@ -0,0 +1,88 @@ +// Step 3: Download Piece +// This step downloads the piece using piece CID from step 2 +// Set before running: +// PDP_URL=https://andyserver.thepianoexpress.com +// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet +// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode +// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode +// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 +// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... +// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 + +import { getAuthConfigFromEnv, buildAuthHeader, createClient } from './auth'; + +async function run(pieceCid?: string) { + console.log('šŸ“¦ Step 3: Downloading Piece'); + console.log(' REQUIRED INPUT: Piece CID from Step 2'); + + // Get configuration from environment + const config = getAuthConfigFromEnv(); + + // Build authentication + const authHeader = await buildAuthHeader(config); + + // Create authenticated client + const client = createClient(config, authHeader); + + // Use provided pieceCid or get from environment + const targetPieceCid = pieceCid || process.env.PIECE_CID; + if (!targetPieceCid) { + console.error('āŒ REQUIRED INPUT MISSING: Piece CID'); + console.error(' This step requires a piece CID from Step 2.'); + console.error(' Either pass as parameter: run("your-piece-cid")'); + console.error(' Or set environment variable: export PIECE_CID=your-piece-cid'); + console.error(''); + console.error(' To get the piece CID, run Step 2 first:'); + console.error(' npx ts-node 2.ts'); + throw new Error('REQUIRED INPUT MISSING: Piece CID from Step 2'); + } + + console.log(` Using piece CID: ${targetPieceCid}`); + + // Retrieve piece via market server + console.log('šŸ“¦ Retrieving piece via market server...'); + try { + const base = config.serverUrl.replace(/\/$/, ''); + const url = `${base}/piece/${targetPieceCid}`; + console.log(` Retrieval URL: ${url}`); + + const r = await fetch(url); + console.log(` Retrieval HTTP status: ${r.status}`); + + if (r.ok) { + const retrieved = new Uint8Array(await r.arrayBuffer()); + console.log(` Retrieved ${retrieved.length} bytes`); + console.log('āœ… Content retrieval: SUCCESS'); + + return { + pieceCid: targetPieceCid, + retrievedBytes: retrieved, + success: true, + }; + } else { + const errorText = await r.text().catch(() => ''); + console.log(` Retrieval failed with status ${r.status}: ${errorText}`); + return { + pieceCid: targetPieceCid, + success: false, + error: `HTTP ${r.status}: ${errorText}`, + }; + } + } catch (e) { + console.warn(' Retrieval attempt failed:', (e as Error).message); + return { + pieceCid: targetPieceCid, + success: false, + error: (e as Error).message, + }; + } +} + +if (require.main === module) { + run().catch(err => { + console.error('Step 3 failed:', err); + process.exit(1); + }); +} + +export { run as downloadPiece }; diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts new file mode 100644 index 000000000..7d5a225c5 --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts @@ -0,0 +1,110 @@ +// Step 4: Delete +// This step deletes using upload ID from step 2 +// Set before running: +// PDP_URL=https://andyserver.thepianoexpress.com +// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet +// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode +// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode +// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 +// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... +// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 + +import { getAuthConfigFromEnv, buildAuthHeader, createClient } from './auth'; +import { Mk20Deal, Mk20Products, Mk20PDPV1 } from '../../generated'; + +async function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function run(uploadId?: string) { + console.log('šŸ—‘ļø Step 4: Deleting Deal'); + console.log(' REQUIRED INPUT: Upload ID from Step 2'); + + // Get configuration from environment + const config = getAuthConfigFromEnv(); + + // Build authentication + const authHeader = await buildAuthHeader(config); + + // Create authenticated client + const client = createClient(config, authHeader); + + // Use provided uploadId or get from environment + const targetUploadId = uploadId || process.env.UPLOAD_ID; + if (!targetUploadId) { + console.error('āŒ REQUIRED INPUT MISSING: Upload ID'); + console.error(' This step requires an upload ID from Step 2.'); + console.error(' Either pass as parameter: run("upload-id")'); + console.error(' Or set environment variable: export UPLOAD_ID=your-upload-id'); + console.error(''); + console.error(' To get the upload ID, run Step 2 first:'); + console.error(' npx ts-node 2.ts'); + throw new Error('REQUIRED INPUT MISSING: Upload ID from Step 2'); + } + + console.log(` Using upload ID: ${targetUploadId}`); + + console.log(`šŸ—‘ļø Requesting deletion of upload ${targetUploadId}...`); + + // Request deletion by updating the deal with delete flags + const deleteDeal: Mk20Deal = { + identifier: targetUploadId, + client: config.clientAddr, + products: { + pdpV1: { + deletePiece: true, + deleteDataSet: true, + recordKeeper: config.recordKeeper, + } as Mk20PDPV1 + } as Mk20Products + }; + + try { + const result = await client.updateDeal(targetUploadId, deleteDeal); + console.log(` Deletion request submitted successfully: ${result}`); + } catch (e) { + console.error('Deletion request error:', (e as Error).message); + try { + const re: any = e as any; + if (re && re.response) { + const status = re.response.status; + const text = await re.response.text().catch(() => ''); + console.error('Deletion error status:', status); + console.error('Deletion error body:', text); + } + } catch (_) {} + throw e; + } + + // Poll deal status post-deletion + console.log('ā³ Polling deal status post-deletion...'); + for (let i = 0; i < 24; i++) { // up to ~2 minutes + const status = await client.getStatus(targetUploadId); + const pdp = status.pdpV1; + const st = pdp?.status; + console.log(` Status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); + if (st === 'complete' || st === 'failed') break; + await sleep(5000); + } + + const finalStatus = await client.getStatus(targetUploadId); + console.log('āœ… Step 4 completed: Deal deletion finished'); + console.log(` - Deleted upload ID: ${targetUploadId}`); + console.log(` - Final status: ${finalStatus.pdpV1?.status}`); + console.log(''); + console.log('šŸŽ‰ All steps completed! End-to-end workflow finished successfully.'); + + return { + deletedUploadId: targetUploadId, + finalStatus: finalStatus.pdpV1?.status, + }; +} + +if (require.main === module) { + run().catch(err => { + console.error('Step 4 failed:', err); + process.exit(1); + }); +} + +export { run as deleteDeal }; diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/README.md b/market/mk20/tsclient/examples/unpkg-end-to-end/README.md new file mode 100644 index 000000000..db9c4cf6d --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/README.md @@ -0,0 +1,157 @@ +# Unpkg End-to-End Example + +This folder contains a step-by-step example of the complete PDPv1 workflow, broken down into individual steps that run in order and use output from previous steps. + +## Prerequisites + +Set the following environment variables before running any step: + +```bash +# Required +export PDP_URL=https://your-server.com +export PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai # client wallet +export PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 +export PDP_RECORD_KEEPER=t1000 # record keeper address (required for PDPv1) + +# For ed25519 authentication (default) +export PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes +export PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 +export PDP_KEY_TYPE=ed25519 + +# OR for secp256k1 authentication +export PDP_KEY_TYPE=secp256k1 +export PDP_SECP_PRIVATE_KEY_HEX=your_32_byte_hex_key +# OR +export PDP_SECP_PRIVATE_KEY_B64=your_32_byte_base64_key + +# Optional +export PDP_INSECURE_TLS=1 # Only for debugging - disables TLS verification +``` + +## Steps (Run in Order) + +### 1. Create Dataset (`1.ts`) +Creates a PDPv1 dataset (first part of `startPDPv1DealForUpload`). + +```bash +npx ts-node 1.ts +``` + +**What it does:** +- Creates a PDPv1 dataset with `createDataSet: true` +- Uses `submitDeal` API with dataset creation deal +- Waits for dataset creation to complete +- Returns dataset ID for use in step 2 + +**Output:** Dataset ID that should be passed to step 2 + +### 2. Add Piece (`2.ts`) +Adds a piece to the dataset (second part of `startPDPv1DealForUpload`). + +```bash +# Set the dataset ID from step 1 +export DATASET_ID=your_dataset_id_from_step_1 +npx ts-node 2.ts +``` + +**What it does:** +- Downloads file from unpkg to compute piece CID +- Creates add piece deal with `addPiece: true` and `dataSetId` +- Uses `submitDeal` API with add piece deal +- Waits for add piece to complete +- Returns upload ID, deal ID, and piece CID + +**Output:** Upload ID, deal ID, piece CID, and deal object for step 3 + +### 3. Upload Blobs (`3.ts`) +Uploads blobs using the deal from step 2. + +```bash +# Use the upload ID and deal from step 2 +export UPLOAD_ID=your_upload_id_from_step_2 +npx ts-node 3.ts +``` + +**What it does:** +- Uses `uploadBlobs` API with the deal from step 2 +- Uploads the file data to the deal +- Monitors upload progress until completion + +**Output:** Upload completion status + +### 4. Download Piece (`4.ts`) +Downloads the piece using piece CID from step 2. + +```bash +# Use the piece CID from step 2 +export PIECE_CID=your_piece_cid_from_step_2 +npx ts-node 4.ts +``` + +**What it does:** +- Retrieves the uploaded piece via market server +- Uses the piece CID provided from step 2 +- Verifies successful retrieval + +**Output:** Retrieved content and success status + +### 5. Delete (`5.ts`) +Deletes using upload ID from step 3. + +```bash +# Use the upload ID from step 3 +export UPLOAD_ID=your_upload_id_from_step_3 +npx ts-node 5.ts +``` + +**What it does:** +- Updates the deal with `deletePiece: true` and `deleteDataSet: true` +- Uses `updateDeal` API to request deletion +- Monitors deletion progress +- Completes the end-to-end workflow + +**Output:** Deletion confirmation and final status + +## Running the Complete Workflow + +You can run all steps in sequence, passing data between them: + +```bash +# Step 1: Create dataset +DATASET_ID=$(npx ts-node 1.ts | grep "Dataset ID:" | cut -d' ' -f3) + +# Step 2: Add piece +export DATASET_ID +UPLOAD_ID=$(npx ts-node 2.ts | grep "Upload ID:" | cut -d' ' -f3) +PIECE_CID=$(npx ts-node 2.ts | grep "Piece CID:" | cut -d' ' -f3) + +# Step 3: Upload blobs +export UPLOAD_ID +npx ts-node 3.ts + +# Step 4: Download piece +export PIECE_CID +npx ts-node 4.ts + +# Step 5: Delete +export UPLOAD_ID +npx ts-node 5.ts +``` + +## Files + +- `auth.ts` - Authentication helpers and configuration management +- `1.ts` - Create dataset step +- `2.ts` - Add piece step +- `3.ts` - Upload blobs step +- `4.ts` - Download piece step +- `5.ts` - Delete step +- `README.md` - This documentation + +## Notes + +- **Each step builds on the previous**: Steps are designed to run in order and use output from previous steps +- **Environment variables**: Use `DATASET_ID`, `UPLOAD_ID`, and `PIECE_CID` environment variables to pass data between steps +- **Matches startPDPv1DealForUpload**: Steps 1-2 replicate the internal logic of `startPDPv1DealForUpload` function +- **Real workflow**: This demonstrates the actual API calls you'd make in a production environment +- **Error handling**: All steps include comprehensive error handling and status reporting diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/auth.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/auth.ts new file mode 100644 index 000000000..e825f8d40 --- /dev/null +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/auth.ts @@ -0,0 +1,42 @@ +// Re-export all auth utilities from the main src module +export { + AuthConfig, + buildAuthHeader, + createClient, + sanitizeAuthHeader, + runPreflightChecks +} from '../../src/auth'; + +/** + * Get authentication configuration from environment variables + * This is the only environment-specific function that stays in examples + */ +export function getAuthConfigFromEnv(): import('../../src/auth').AuthConfig { + if (process.env.PDP_INSECURE_TLS === '1') { + // Disable TLS verification (use only for debugging!) + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + console.warn('WARNING: PDP_INSECURE_TLS=1 set. TLS verification disabled.'); + } + + const keyType = (process.env.PDP_KEY_TYPE || 'ed25519').toLowerCase() as 'ed25519' | 'secp256k1'; + + const recordKeeper = process.env.PDP_RECORD_KEEPER; + if (!recordKeeper) { + console.error('āŒ REQUIRED ENVIRONMENT VARIABLE MISSING: PDP_RECORD_KEEPER'); + console.error(' The record keeper is required for PDPv1 deals.'); + console.error(' Set it with: export PDP_RECORD_KEEPER=your-record-keeper-address'); + throw new Error('REQUIRED ENVIRONMENT VARIABLE MISSING: PDP_RECORD_KEEPER'); + } + + return { + serverUrl: process.env.PDP_URL || 'http://localhost:8080', + clientAddr: process.env.PDP_CLIENT || 'f1client...', + recordKeeper, + contractAddress: process.env.PDP_CONTRACT || '0x0000000000000000000000000000000000000000', + keyType, + publicKeyB64: process.env.PDP_PUBLIC_KEY_B64, + privateKeyB64: process.env.PDP_PRIVATE_KEY_B64, + secpPrivateKeyHex: process.env.PDP_SECP_PRIVATE_KEY_HEX, + secpPrivateKeyB64: process.env.PDP_SECP_PRIVATE_KEY_B64, + }; +} diff --git a/market/mk20/tsclient/src/auth.ts b/market/mk20/tsclient/src/auth.ts index 99fb6ab2c..295c92ec0 100644 --- a/market/mk20/tsclient/src/auth.ts +++ b/market/mk20/tsclient/src/auth.ts @@ -205,6 +205,19 @@ export class AuthUtils { export default AuthUtils; +// Configuration interface for authentication +export interface AuthConfig { + serverUrl: string; + clientAddr: string; + recordKeeper: string; + contractAddress: string; + keyType: 'ed25519' | 'secp256k1'; + publicKeyB64?: string; + privateKeyB64?: string; + secpPrivateKeyHex?: string; + secpPrivateKeyB64?: string; +} + /** Generic signer interface */ export interface AuthSigner { getPublicKey(): Promise | Uint8Array; @@ -316,4 +329,97 @@ export class Secp256k1AddressSigner implements AuthSigner { } } +// Utility functions for authentication and client management + +/** + * Build authentication header from configuration + */ +export async function buildAuthHeader(config: AuthConfig): Promise { + if (config.keyType === 'ed25519') { + if (!config.publicKeyB64 || !config.privateKeyB64) { + throw new Error('PDP_PUBLIC_KEY_B64 and PDP_PRIVATE_KEY_B64 must be set for ed25519'); + } + const pub = Uint8Array.from(Buffer.from(config.publicKeyB64, 'base64')); + const priv = Uint8Array.from(Buffer.from(config.privateKeyB64, 'base64')); + const signer = new Ed25519KeypairSigner(pub, priv); + return await AuthUtils.buildAuthHeader(signer, 'ed25519'); + } else if (config.keyType === 'secp256k1') { + // Derive pubKeyBase64 from Filecoin address bytes + const addrBytes = Secp256k1AddressSigner.addressBytesFromString(config.clientAddr); + const pubB64 = Buffer.from(addrBytes).toString('base64'); + if (!pubB64) throw new Error('Unable to derive address bytes from PDP_CLIENT'); + + // Load secp256k1 private key from env (HEX preferred, else B64) + let priv: Uint8Array | undefined; + if (config.secpPrivateKeyHex) { + const clean = config.secpPrivateKeyHex.startsWith('0x') ? config.secpPrivateKeyHex.slice(2) : config.secpPrivateKeyHex; + if (clean.length !== 64) throw new Error('PDP_SECP_PRIVATE_KEY_HEX must be 32-byte (64 hex chars)'); + const bytes = new Uint8Array(32); + for (let i = 0; i < 32; i++) bytes[i] = parseInt(clean.substr(i * 2, 2), 16); + priv = bytes; + } else if (config.secpPrivateKeyB64) { + const buf = Buffer.from(config.secpPrivateKeyB64, 'base64'); + if (buf.length !== 32) throw new Error('PDP_SECP_PRIVATE_KEY_B64 must decode to 32 bytes'); + priv = new Uint8Array(buf); + } + if (!priv) throw new Error('Set PDP_SECP_PRIVATE_KEY_HEX or PDP_SECP_PRIVATE_KEY_B64 for secp256k1 signing'); + + // Use Secp256k1AddressSigner (address bytes derived from PDP_CLIENT) + const signer = new Secp256k1AddressSigner(config.clientAddr, priv); + return await AuthUtils.buildAuthHeader(signer, 'secp256k1'); + } else { + throw new Error(`Unsupported PDP_KEY_TYPE: ${config.keyType}`); + } +} + +/** + * Create authenticated client from configuration and auth header + */ +export function createClient(config: AuthConfig, authHeader: string): any { + const clientConfig = { + serverUrl: config.serverUrl, + headers: { Authorization: authHeader }, + }; + // Use the same pattern as the original unpkg-end-to-end.ts file + return new (require('./client').MarketClient)(clientConfig); +} + +/** + * Sanitize auth header for logging (removes sensitive signature data) + */ +export function sanitizeAuthHeader(authHeader: string): string { + return authHeader.replace(/:[A-Za-z0-9+/=]{16,}:/, (m) => `:${m.slice(1, 9)}...:`); +} + +/** + * Run preflight connectivity checks + */ +export async function runPreflightChecks(config: AuthConfig, authHeader: string): Promise { + try { + const base = config.serverUrl.replace(/\/$/, ''); + const urls: Array<{ url: string; headers?: Record }> = [ + { url: `${base}/health` }, + { url: `${base}/market/mk20/info/swagger.json` }, + { url: `${base}/market/mk20/products`, headers: { Authorization: authHeader } }, + ]; + + for (const { url, headers } of urls) { + try { + const init: RequestInit = headers ? { headers } : {}; + const r = await fetch(url, init); + console.log(`Preflight ${url}:`, r.status); + if (!r.ok) { + const text = await r.text().catch(() => ''); + console.log(`Preflight body (${url}):`, text); + } + } catch (e) { + const err = e as any; + console.error(`Preflight failed (${url}):`, err?.message || String(e), err?.cause?.code || '', err?.code || ''); + } + } + } catch (e) { + console.error('Preflight orchestrator failed:', (e as Error).message); + } +} + diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index 405618653..56424b085 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -278,6 +278,7 @@ export class PieceCidUtils { export class MarketClient { private api: DefaultApi; + private config: MarketClientConfig; /** * Try to extract a human-friendly error string from an HTTP Response. @@ -305,6 +306,7 @@ export class MarketClient { * @param config.fetchApi - Optional custom fetch implementation */ constructor(config: MarketClientConfig) { + this.config = config; const basePath = `${config.serverUrl.replace(/\/$/, '')}/market/mk20`; const runtimeConfig = { ...config, basePath } as ConfigurationParameters; this.api = new DefaultApi(new Configuration(runtimeConfig)); @@ -326,10 +328,11 @@ export class MarketClient { } /** - * Convert a ULID string (26-char Crockford base32) into a 16-byte array + * Convert a ULID string (26-char Crockford base32) into an ASCII byte array */ private ulidToBytes(ulidString: string): number[] { - var bytes: number[] = []; + // ULID is 26 characters, convert to ASCII byte array + const bytes: number[] = []; for (let i = 0; i < ulidString.length; i++) { bytes.push(ulidString.charCodeAt(i)); } @@ -503,28 +506,31 @@ export class MarketClient { await this.submitDeal(createDeal); await this.waitDealComplete(datasetId); + var datasetIdNumber = 0; // TODO: get dataset id from response + // Step 2: add piece with data under a new identifier (upload id) const uploadId = ulid(); const addPieceDeal: Mk20Deal = { identifier: uploadId, client, data: { - pieceCid: pieceCid, + pieceCid: { "/": pieceCid } as object, format: { raw: {} }, sourceHttpPut: {}, } as Mk20DataSource, products: { pdpV1: { addPiece: true, + dataSetId: datasetIdNumber, recordKeeper: recordKeeper, extraData: [], deleteDataSet: false, deletePiece: false, } as Mk20PDPV1, retrievalV1: { - announcePayload: true, + announcePayload: false, // not a CAR file. announcePiece: true, - indexing: true, + indexing: false, // not a CAR file. } as Mk20RetrievalV1, } as Mk20Products, } as Mk20Deal; @@ -557,7 +563,7 @@ export class MarketClient { const chunk = blob.slice(offset, offset + chunkSize); const chunkArray = new Uint8Array(await chunk.arrayBuffer()); const chunkNumbers = Array.from(chunkArray); - const chunkNum = String(totalChunks); + const chunkNum = String(totalChunks + 1); await this.uploadChunk(id, chunkNum, chunkNumbers); totalChunks++; uploadedBytes += chunkNumbers.length; @@ -603,7 +609,7 @@ export class MarketClient { totalSize: prep.totalSize, dealId: prep.dealId, uploadId: prep.id, - pieceCid: prep.pieceCid, + pieceCid: prep.pieceCid , uploadedChunks: ures.uploadedChunks, uploadedBytes: ures.uploadedBytes, }; @@ -670,7 +676,12 @@ export class MarketClient { */ async uploadChunk(id: string, chunkNum: string, data: Array): Promise { try { - const apiResp = await this.api.uploadsIdChunkNumPutRaw({ id, chunkNum, data }); + const apiResp = await this.api.uploadsIdChunkNumPutRaw({ id, chunkNum, data }, { + headers: { + 'Content-Type': 'application/octet-stream', + 'Authorization': this.config.headers?.Authorization || '' + } + }); const ct = apiResp.raw.headers.get('content-type') || ''; if (ct.toLowerCase().includes('application/json') || ct.toLowerCase().includes('+json')) { return await apiResp.value(); diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts index b1d79f8ab..eba78a191 100644 --- a/market/mk20/tsclient/src/streaming.ts +++ b/market/mk20/tsclient/src/streaming.ts @@ -240,7 +240,7 @@ export class StreamingPDP { const pieceCid = StreamingCommP.pieceCidV2FromDigest(this.totalSize, digest); const dataSource: Mk20DataSource = { - pieceCid: pieceCid, + pieceCid: { "/": pieceCid } as { [key: string]: string; }, format: { raw: {} } as Mk20PieceDataFormat, sourceHttpPut: { raw_size: this.totalSize } as unknown as object, }; diff --git a/market/mk20/tsclient/tsconfig.json b/market/mk20/tsclient/tsconfig.json index 8828dcc83..7e9d3aa9b 100644 --- a/market/mk20/tsclient/tsconfig.json +++ b/market/mk20/tsclient/tsconfig.json @@ -3,6 +3,7 @@ "target": "ES2020", "module": "commonjs", "lib": ["ES2020", "DOM"], + "types": ["node"], "declaration": true, "outDir": "./dist", "rootDir": ".", diff --git a/market/mk20/types.go b/market/mk20/types.go index 74909022f..9014f123f 100644 --- a/market/mk20/types.go +++ b/market/mk20/types.go @@ -14,7 +14,7 @@ import ( type Deal struct { // Identifier represents a unique identifier for the deal in ULID format. - Identifier ulid.ULID `json:"identifier"` + Identifier ulid.ULID `json:"identifier" swaggertype:"string" format:"ulid" example:"01ARZ3NDEKTSV4RRFFQ69G5FAV"` // Client wallet string for the deal Client string `json:"client"` @@ -41,7 +41,7 @@ type Products struct { type DataSource struct { // PieceCID represents the unique identifier (pieceCID V2) for a piece of data, stored as a CID object. - PieceCID cid.Cid `json:"piece_cid"` + PieceCID cid.Cid `json:"piece_cid" swaggertype:"object,string" example:"/:bafkzcibfxx3meais3xzh6qn56y6hiasmrufhegoweu3o5ccofs74nfdfr4yn76pqz4pq"` // Format defines the format of the piece data, which can include CAR, Aggregate, or Raw formats. Format PieceDataFormat `json:"format"` From 5fe3e82ee72e6cc6b3f51eefeac09439007072b2 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 10 Sep 2025 18:12:30 -0500 Subject: [PATCH 47/55] packageJson update --- market/mk20/tsclient/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index d528a30e3..e2d717e31 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -1,9 +1,9 @@ { "name": "@curio/market-client", - "version": "1.0.0", - "description": "TypeScript API client for Curio storage market", - "main": "dist/index.js", - "types": "dist/index.d.ts", + "version": "0.4.0", + "description": "TypeScript API client for Curio storage market. You probably want the Synapse SDK instead.", + "main": "dist/src/index.js", + "types": "dist/src/index.d.ts", "scripts": { "build": "npm run generate && npm run compile", "generate": "openapi-generator-cli generate -i ../http/swagger.json -g typescript-fetch -o ./generated --additional-properties=supportsES6=true,typescriptThreePlus=true --skip-validate-spec", From 3523a02ce21ea1b144ba0b73c49213955a10e17d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 10 Sep 2025 20:48:12 -0500 Subject: [PATCH 48/55] naming --- market/mk20/tsclient/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index e2d717e31..0a660ee87 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -1,5 +1,5 @@ { - "name": "@curio/market-client", + "name": "@curiostorage/market-client", "version": "0.4.0", "description": "TypeScript API client for Curio storage market. You probably want the Synapse SDK instead.", "main": "dist/src/index.js", From 80d989647891a4390f9913b726a5f38d4ec741c3 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 10 Sep 2025 21:04:16 -0500 Subject: [PATCH 49/55] readme --- market/mk20/tsclient/README.md | 241 ++++++++++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 5 deletions(-) diff --git a/market/mk20/tsclient/README.md b/market/mk20/tsclient/README.md index f7f435cf7..e496db0ac 100644 --- a/market/mk20/tsclient/README.md +++ b/market/mk20/tsclient/README.md @@ -5,9 +5,108 @@ This is a TypeScript API client for the Curio storage market API. It provides a ## Installation ```bash -npm install @curio/market-client +npm install @curiostorage/market-client ``` +## Prerequisites + +**Authentication is required** for all API operations. You must configure authentication before using the client. + +### Authentication Methods + +The client supports two authentication methods: + +1. **Ed25519** (default) - Uses Ed25519 key pairs +2. **Secp256k1** - Uses Secp256k1 key pairs (compatible with Ethereum wallets) + +### Authentication Configuration + +Authentication can be configured programmatically or via environment variables (used in examples): + +**Programmatic Configuration:** +```typescript +const authConfig = { + serverUrl: 'https://your-server.com', + clientAddr: 'f1client...', + recordKeeper: 't1000', // Required for PDPv1 deals + contractAddress: '0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6', + keyType: 'ed25519' as 'ed25519' | 'secp256k1', + publicKeyB64: 'your_base64_public_key', + privateKeyB64: 'your_base64_private_key', + // OR for secp256k1: + // secpPrivateKeyHex: 'your_hex_private_key', + // secpPrivateKeyB64: 'your_base64_private_key', +}; +``` + +**Environment Variables (for examples):** +```bash +# Used in the example scripts +export PDP_URL=https://your-server.com +export PDP_CLIENT=f1client... +export PDP_RECORD_KEEPER=t1000 +export PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 +export PDP_KEY_TYPE=ed25519 +export PDP_PUBLIC_KEY_B64=your_base64_public_key +export PDP_PRIVATE_KEY_B64=your_base64_private_key +``` + +**Running Example Scripts:** +```bash +# Example: Running step 1 with all environment variables inline +PDP_INSECURE_TLS=1 \ +PDP_URL="https://your-server.com" \ +PDP_CLIENT=f1client... \ +PDP_KEY_TYPE=secp256k1 \ +PDP_SECP_PRIVATE_KEY_B64="your_base64_private_key" \ +PDP_CONTRACT="0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6" \ +PDP_RECORD_KEEPER="0x158c8f05A616403589b99BE5d82d756860363A92" \ +DATASET_ID="01K4TKYS9302Y42BRBT0V0S389" \ +npx ts-node 1.ts +``` + +You'll need a wallet: +Ā lotus wallet new delegated + +You can get your private key (for demo only) from: + lotus wallet export | xxd -r -p | jq -r ā€˜.PrivateKeyā€™ | base64 -d | xxd -p -c 32 + +Be sure to setup Curio with: +- ui ā€”> pdp ā€”> ownerAddress ā€”> (hex key) +- Your curio also needs storage attached: + -- ./curio cli storage attach -snap -seal /home/ubuntu/curiofolder + -- And a market enabled, such as taking the following with ./curio config set market.yaml +market.yaml: +[Batching] + [Batching.Commit] + Timeout = "0h0m5s" + [Batching.PreCommit] + Slack = "6h0m0s" + Timeout = "0h0m5s" + +[HTTP] + DelegateTLS = false + DomainName = "yourserver.yourdomain.com" + Enable = true + ListenAddress = "0.0.0.0:443" + +[Ingest] + MaxDealWaitTime = "0h0m30s" + +[Market] + [Market.StorageMarketConfig] + [Market.StorageMarketConfig.MK12] + ExpectedPoRepSealDuration = "0h1m0s" + ExpectedSnapSealDuration = "0h1m0s" + PublishMsgPeriod = "0h0m10s" + +[Subsystems] + EnableCommP = true + EnableDealMarket = true + EnablePDP = true + EnableParkPiece = true + + ## Building from Source 1. Install dependencies: @@ -33,9 +132,27 @@ npm run build ## Usage ```typescript -import { MarketClient, PieceCidUtils } from '@curio/market-client'; +import { MarketClient, PieceCidUtils, AuthUtils } from '@curiostorage/market-client'; + +// Configure authentication programmatically +const authConfig = { + serverUrl: 'https://your-server.com', + clientAddr: 'f1client...', + recordKeeper: 't1000', // Required for PDPv1 deals + contractAddress: '0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6', + keyType: 'ed25519' as 'ed25519' | 'secp256k1', + publicKeyB64: 'your_base64_public_key', + privateKeyB64: 'your_base64_private_key', +}; -const client = new MarketClient({ serverUrl: 'http://localhost:8080' }); +// Build authentication header +const authHeader = await AuthUtils.buildAuthHeader(authConfig); + +// Create authenticated client +const client = new MarketClient({ + serverUrl: authConfig.serverUrl, + authHeader +}); // Get supported contracts const contracts = await client.getContracts(); @@ -99,9 +216,25 @@ console.log('Uploaded bytes:', result.uploadedBytes); Create a deal without a `data` section, stream data using `uploadChunk`, compute the piece CID while streaming, then finalize with the computed `data`: ```typescript -import { Client, MarketClientConfig } from '@curio/market-client'; +import { Client, MarketClientConfig, AuthUtils } from '@curiostorage/market-client'; + +// Configure authentication programmatically +const authConfig = { + serverUrl: 'https://your-server.com', + clientAddr: 'f1client...', + recordKeeper: 't1000', // Required for PDPv1 deals + contractAddress: '0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6', + keyType: 'ed25519' as 'ed25519' | 'secp256k1', + publicKeyB64: 'your_base64_public_key', + privateKeyB64: 'your_base64_private_key', +}; + +const authHeader = await AuthUtils.buildAuthHeader(authConfig); -const config: MarketClientConfig = { serverUrl: 'http://localhost:8080' }; +const config: MarketClientConfig = { + serverUrl: authConfig.serverUrl, + authHeader +}; const client = new Client(config); // Create the streaming helper (defaults to 1MB chunks) @@ -253,6 +386,104 @@ Converts an existing CID v1 to piece CID v2 format, supporting: - Filecoin sealed commitments (Poseidon) - Raw data codecs +## Troubleshooting + +### Common Authentication Issues + +**Error: "REQUIRED ENVIRONMENT VARIABLE MISSING: PDP_RECORD_KEEPER"** +- This error only appears when running the example scripts +- The `PDP_RECORD_KEEPER` environment variable is required for the examples +- Set it with: `export PDP_RECORD_KEEPER=your-record-keeper-address` +- For programmatic usage, include `recordKeeper` in your auth config + +**Error: "Authentication failed" or "Invalid signature"** +- Verify your private key is correctly formatted (base64 or hex) +- Ensure the key type matches your key format (`ed25519` or `secp256k1`) +- Check that the public key corresponds to the private key + +**Error: "TLS verification failed"** +- For debugging only, you can disable TLS verification with `export PDP_INSECURE_TLS=1` +- **Warning**: Never use this in production + +**Error: "Connection refused" or "Network error"** +- Verify the `PDP_URL` is correct and accessible +- Check that the server is running and accepting connections +- Ensure firewall settings allow the connection + +### Key Generation + +**Ed25519 Key Generation:** +```bash +# Generate Ed25519 key pair +node -e " +const crypto = require('crypto'); +const keyPair = crypto.generateKeyPairSync('ed25519'); +console.log('Public key (base64):', keyPair.publicKey.export({ type: 'spki', format: 'der' }).toString('base64')); +console.log('Private key (base64):', keyPair.privateKey.export({ type: 'pkcs8', format: 'der' }).toString('base64')); +" +``` + +**Secp256k1 Key Generation:** +```bash +# Generate Secp256k1 key pair +node -e " +const crypto = require('crypto'); +const keyPair = crypto.generateKeyPairSync('ec', { namedCurve: 'secp256k1' }); +console.log('Private key (hex):', keyPair.privateKey.export({ type: 'sec1', format: 'der' }).toString('hex')); +console.log('Private key (base64):', keyPair.privateKey.export({ type: 'sec1', format: 'der' }).toString('base64')); +" +``` + +### Environment Variable Validation + +The example client validates your configuration: + +```typescript +import { AuthUtils } from '@curiostorage/market-client'; + +const authConfig = { + // ... your config +}; + +try { + const authHeader = await AuthUtils.buildAuthHeader(authConfig); + console.log('āœ… Authentication configuration is valid'); +} catch (error) { + console.error('āŒ Authentication configuration error:', error.message); +} +``` + +## Examples + +See the `examples/unpkg-end-to-end/` directory for a complete step-by-step workflow that demonstrates: + +- Authentication setup and configuration (using environment variables) +- Creating PDPv1 datasets +- Adding pieces to datasets +- Uploading data with chunked uploads +- Downloading pieces +- Deleting datasets and pieces + +Each step is documented and can be run independently, making it easy to understand the complete workflow. The examples use environment variables for configuration, but you can adapt the code to use programmatic configuration instead. + +**Quick Start with Examples:** +```bash +# Set your configuration +export PDP_URL="https://your-server.com" +export PDP_CLIENT="f1client..." +export PDP_RECORD_KEEPER="0x158c8f05A616403589b99BE5d82d756860363A92" +export PDP_CONTRACT="0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6" +export PDP_KEY_TYPE="secp256k1" +export PDP_SECP_PRIVATE_KEY_B64="your_base64_private_key" + +# Run the complete workflow +cd examples/unpkg-end-to-end/ +npx ts-node 1.ts # Create dataset +npx ts-node 2.ts # Add piece and upload +npx ts-node 3.ts # Download piece +npx ts-node 4.ts # Delete +``` + ## Development The client is generated from the OpenAPI/Swagger specification in `../http/swagger.json`. To regenerate after API changes: From 68157cf6a27121b20f4fbd1b9efdfb67978e9df6 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 10 Sep 2025 21:05:08 -0500 Subject: [PATCH 50/55] version --- market/mk20/tsclient/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 0a660ee87..1d7cf687a 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -1,6 +1,6 @@ { "name": "@curiostorage/market-client", - "version": "0.4.0", + "version": "0.4.1", "description": "TypeScript API client for Curio storage market. You probably want the Synapse SDK instead.", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", From 699e04310ac3bb043549a5c9c6ae6f7cc687b18d Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 15 Sep 2025 10:19:33 -0500 Subject: [PATCH 51/55] progress --- .../tsclient/examples/unpkg-end-to-end.ts | 205 ------------------ .../tsclient/examples/unpkg-end-to-end/1.ts | 5 - .../tsclient/examples/unpkg-end-to-end/2.ts | 23 -- 3 files changed, 233 deletions(-) delete mode 100644 market/mk20/tsclient/examples/unpkg-end-to-end.ts diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end.ts b/market/mk20/tsclient/examples/unpkg-end-to-end.ts deleted file mode 100644 index 8467a6509..000000000 --- a/market/mk20/tsclient/examples/unpkg-end-to-end.ts +++ /dev/null @@ -1,205 +0,0 @@ -// Set before running: -// PDP_URL=https://andyserver.thepianoexpress.com -// PDP_CLIENT=t1k7ctd3hvmwwjdpb2ipd3kr7n4vk3xzfvzbbdrai // client wallet -// PDP_PUBLIC_KEY_B64=base64_of_raw_public_key_32_bytes # ed25519 mode -// PDP_PRIVATE_KEY_B64=base64_of_secret_key_64_or_seed_32 # ed25519 mode -// PDP_KEY_TYPE=ed25519|secp256k1 # default ed25519 -// PDP_SECP_PRIVATE_KEY_HEX=... or PDP_SECP_PRIVATE_KEY_B64=... -// PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 -import { Client, MarketClientConfig, PieceCidUtils, AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from '../src'; - -async function downloadFromUnpkg(url: string): Promise { - const res = await fetch(url); - if (!res.ok) throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); - const buf = await res.arrayBuffer(); - return new Uint8Array(buf); -} - -async function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -async function run() { - if (process.env.PDP_INSECURE_TLS === '1') { - // Disable TLS verification (use only for debugging!) - process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; - console.warn('WARNING: PDP_INSECURE_TLS=1 set. TLS verification disabled.'); - } - - const config: MarketClientConfig = { - serverUrl: process.env.PDP_URL || 'http://localhost:8080', - } as MarketClientConfig; - const clientAddr = process.env.PDP_CLIENT || 'f1client...'; - const recordKeeper = process.env.PDP_RECORD_KEEPER || 't1000'; - const contractAddress = process.env.PDP_CONTRACT || '0x0000000000000000000000000000000000000000'; - - // Build Authorization header - const keyType = (process.env.PDP_KEY_TYPE || 'ed25519').toLowerCase(); - let authHeader: string; - if (keyType === 'ed25519') { - const pubB64 = process.env.PDP_PUBLIC_KEY_B64 || ''; - const privB64 = process.env.PDP_PRIVATE_KEY_B64 || ''; - if (!pubB64 || !privB64) throw new Error('PDP_PUBLIC_KEY_B64 and PDP_PRIVATE_KEY_B64 must be set for ed25519'); - const pub = Uint8Array.from(Buffer.from(pubB64, 'base64')); - const priv = Uint8Array.from(Buffer.from(privB64, 'base64')); - const signer = new Ed25519KeypairSigner(pub, priv); - authHeader = await AuthUtils.buildAuthHeader(signer, 'ed25519'); - } else if (keyType === 'secp256k1') { - // Derive pubKeyBase64 from Filecoin address bytes - const addrStr = clientAddr; - const { Secp256k1AddressSigner } = require('../src'); - const addrBytes = Secp256k1AddressSigner.addressBytesFromString(addrStr); - const pubB64 = Buffer.from(addrBytes).toString('base64'); - if (!pubB64) throw new Error('Unable to derive address bytes from PDP_CLIENT'); - - // Load secp256k1 private key from env (HEX preferred, else B64) - const privHex = process.env.PDP_SECP_PRIVATE_KEY_HEX || ''; - const privB64 = process.env.PDP_SECP_PRIVATE_KEY_B64 || ''; - let priv: Uint8Array | undefined; - if (privHex) { - const clean = privHex.startsWith('0x') ? privHex.slice(2) : privHex; - if (clean.length !== 64) throw new Error('PDP_SECP_PRIVATE_KEY_HEX must be 32-byte (64 hex chars)'); - const bytes = new Uint8Array(32); - for (let i = 0; i < 32; i++) bytes[i] = parseInt(clean.substr(i * 2, 2), 16); - priv = bytes; - } else if (privB64) { - const buf = Buffer.from(privB64, 'base64'); - if (buf.length !== 32) throw new Error('PDP_SECP_PRIVATE_KEY_B64 must decode to 32 bytes'); - priv = new Uint8Array(buf); - } - if (!priv) throw new Error('Set PDP_SECP_PRIVATE_KEY_HEX or PDP_SECP_PRIVATE_KEY_B64 for secp256k1 signing'); - - // Use Secp256k1AddressSigner (address bytes derived from PDP_CLIENT) - const signer = new Secp256k1AddressSigner(clientAddr, priv); - authHeader = await AuthUtils.buildAuthHeader(signer, 'secp256k1'); - } else { - throw new Error(`Unsupported PDP_KEY_TYPE: ${keyType}`); - } - - const client = new (require('../src').Client)({ - ...config, - headers: { Authorization: authHeader }, - } as MarketClientConfig); - - // Debug: show sanitized auth - const sanitize = (h: string) => h.replace(/:[A-Za-z0-9+/=]{16,}:/, (m) => `:${m.slice(1, 9)}...:`); - console.log('Auth header (sanitized):', sanitize(authHeader)); - console.log('Server URL:', config.serverUrl); - - // Debug: preflight connectivity - try { - const base = config.serverUrl.replace(/\/$/, ''); - const urls: Array<{ url: string; headers?: Record }> = [ - { url: `${base}/health` }, - { url: `${base}/market/mk20/info/swagger.json` }, - { url: `${base}/market/mk20/products`, headers: { Authorization: authHeader } }, - ]; - for (const { url, headers } of urls) { - try { - const init: RequestInit = headers ? { headers } : {}; - const r = await fetch(url, init); - console.log(`Preflight ${url}:`, r.status); - if (!r.ok) { - const text = await r.text().catch(() => ''); - console.log(`Preflight body (${url}):`, text); - } - } catch (e) { - const err = e as any; - console.error(`Preflight failed (${url}):`, err?.message || String(e), err?.cause?.code || '', err?.code || ''); - } - } - } catch (e) { - console.error('Preflight orchestrator failed:', (e as Error).message); - } - - const targetUrl = 'https://unpkg.com/react@18/umd/react.production.min.js'; - console.log(`ā¬‡ļø Downloading: ${targetUrl}`); - const bytes = await downloadFromUnpkg(targetUrl); - console.log(` Downloaded ${bytes.length} bytes`); - - const products = await client.getProducts(); - console.log('Products:', products); - - // Compute piece CID locally for retrieval - const blob = new Blob([Buffer.from(bytes)], { type: 'application/octet-stream' }); - const pieceCid = await PieceCidUtils.computePieceCidV2([blob]); - console.log(`šŸ”— Computed piece CID: ${pieceCid}`); - - console.log('šŸ“Ø Submitting PDPv1 deal and uploading via helper'); - let prep; - try { - prep = await client.startPDPv1DealForUpload({ - blobs: [blob], - client: clientAddr, - recordKeeper: recordKeeper, - contractAddress, - }); - await client.uploadBlobs({ id: prep.id, blobs: [blob], deal: prep.deal }); - } catch (e) { - console.error('Submit error:', (e as Error).message); - try { - const re: any = e as any; - if (re && re.response) { - const status = re.response.status; - const text = await re.response.text().catch(() => ''); - console.error('Submit error status:', status); - console.error('Submit error body:', text); - } - } catch (_) {} - - throw e; - } - const uploadId = prep.id; - - console.log('ā³ Polling deal status until complete/failed'); - for (let i = 0; i < 120; i++) { // up to ~10 minutes with 5s interval - const status = await client.getStatus(uploadId); - const pdp = status.pdpV1; - const st = pdp?.status; - console.log(` status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); - if (st === 'complete' || st === 'failed') break; - await sleep(5000); - } - - console.log('šŸ“¦ Retrieving piece via market server'); - try { - const base = config.serverUrl.replace(/\/$/, ''); - const url = `${base}/piece/${pieceCid}`; - const r = await fetch(url); - console.log(` retrieval HTTP ${r.status}`); - if (r.ok) { - const retrieved = new Uint8Array(await r.arrayBuffer()); - console.log(` retrieved ${retrieved.length} bytes`); - } - } catch (e) { - console.warn(' Retrieval attempt failed:', (e as Error).message); - } - - console.log('šŸ—‘ļø Requesting deletion (set delete flags via update)'); - await client.updateDeal(uploadId, { - identifier: uploadId, - client: clientAddr, - products: { pdpV1: { deletePiece: true, deleteDataSet: true } }, - } as any); - - console.log('ā³ Polling deal status post-deletion'); - for (let i = 0; i < 24; i++) { // up to ~2 minutes - const status = await client.getStatus(uploadId); - const pdp = status.pdpV1; - const st = pdp?.status; - console.log(` status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); - if (st === 'complete' || st === 'failed') break; - await sleep(5000); - } - - console.log('āœ… Example finished'); -} - -if (require.main === module) { - run().catch(err => { - console.error('Example failed:', err); - process.exit(1); - }); -} - -export {}; diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts index 9a424c1cd..2b734a0c8 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts @@ -52,11 +52,6 @@ async function run() { deleteDataSet: false, deletePiece: false, } as Mk20PDPV1, - retrievalV1: { - announcePayload: true, - announcePiece: true, - indexing: true, - } as Mk20RetrievalV1, } as Mk20Products, } as Mk20Deal; diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts index ebeaa34b6..d9dd56c54 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts @@ -104,29 +104,6 @@ async function run(datasetId?: string) { const dealId = await client.submitDeal(addPieceDeal); console.log(` Add piece deal submitted with ID: ${uploadId}, deal ID: ${dealId}`); - // Wait for add piece to complete - console.log('ā³ Waiting for add piece to complete...'); - let addPieceComplete = false; - for (let i = 0; i < 12; i++) { // up to 60 seconds with 5s interval - try { - const status = await client.getStatus(uploadId); - const pdp = status.pdpV1; - const st = pdp?.status; - console.log(` Add piece status: ${st}${pdp?.errorMsg ? `, error: ${pdp.errorMsg}` : ''}`); - if (st === 'complete' || st === 'failed') { - addPieceComplete = true; - break; - } - } catch (e) { - console.log(` Status check failed (attempt ${i + 1}): ${(e as Error).message}`); - if (i === 11) { - console.log(' āš ļø Status polling timed out after 60 seconds'); - break; - } - } - await sleep(5000); - } - if (!addPieceComplete) { console.log(' ā° Add piece status polling timed out after 60 seconds'); console.log(' šŸ”— Please check the blockchain for deal status:'); From 006173a72fa333fff93ade866248ab6651502539 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 15 Sep 2025 16:30:47 -0500 Subject: [PATCH 52/55] toplevel namespace, streaming piece --- .../examples/piece-cid-computation.ts | 23 +- .../tsclient/examples/unpkg-end-to-end/1.ts | 10 +- .../tsclient/examples/unpkg-end-to-end/2.ts | 19 +- .../tsclient/examples/unpkg-end-to-end/4.ts | 8 +- market/mk20/tsclient/src/client.ts | 243 +----------------- market/mk20/tsclient/src/index.ts | 46 +++- market/mk20/tsclient/src/piece.ts | 206 +++++++++++++++ market/mk20/tsclient/src/streaming.ts | 148 ++--------- 8 files changed, 308 insertions(+), 395 deletions(-) create mode 100644 market/mk20/tsclient/src/piece.ts diff --git a/market/mk20/tsclient/examples/piece-cid-computation.ts b/market/mk20/tsclient/examples/piece-cid-computation.ts index ed46f93fc..585052042 100644 --- a/market/mk20/tsclient/examples/piece-cid-computation.ts +++ b/market/mk20/tsclient/examples/piece-cid-computation.ts @@ -1,4 +1,4 @@ -import { PieceCidUtils } from '../src'; +import { CurioMarket } from '../src'; // Example: Compute piece CID v2 from blobs async function computePieceCidExample() { @@ -18,7 +18,7 @@ async function computePieceCidExample() { }); // Compute piece CID v2 - const pieceCid = await PieceCidUtils.computePieceCidV2(mockBlobs); + const pieceCid = CurioMarket.calculatePieceCID(mockBlobs); console.log('\nāœ… Piece CID v2 computed successfully!'); console.log(`šŸ”— Piece CID: ${pieceCid}`); @@ -49,8 +49,8 @@ async function convertCidV1ToV2Example() { console.log(`šŸ” Codec: ${cidV1.code}`); console.log(`šŸ” Hash: ${cidV1.multihash.name}`); - // Convert to piece CID v2 - const pieceCidV2 = await PieceCidUtils.pieceCidV2FromV1(cidV1, mockData.length); + // Convert to piece CID v2 using the better implementation + const pieceCidV2 = CurioMarket.calculatePieceCID(mockData); console.log('\nāœ… Conversion successful!'); console.log(`šŸ“¤ Output piece CID v2: ${pieceCidV2.toString()}`); @@ -83,7 +83,7 @@ async function handleDifferentBlobTypesExample() { }); // Compute piece CID v2 - const pieceCid = await PieceCidUtils.computePieceCidV2(blobs); + const pieceCid = await CurioMarket.PieceCidUtils.computePieceCidV2(blobs); console.log('\nāœ… Piece CID computed for mixed blob types!'); console.log(`šŸ”— Piece CID: ${pieceCid}`); @@ -104,20 +104,19 @@ async function errorHandlingExample() { // Test with empty blob array try { - await PieceCidUtils.computePieceCidV2([]); + await CurioMarket.PieceCidUtils.computePieceCidV2([]); console.log('āŒ Should have thrown error for empty blobs'); } catch (error) { console.log('āœ… Correctly handled empty blob array:', error.message); } - // Test with invalid CID + // Test with invalid data try { - const { CID } = await import('multiformats/cid'); - const invalidCid = CID.create(1, 0x999, { code: 0x999, digest: new Uint8Array(16) }); - await PieceCidUtils.pieceCidV2FromV1(invalidCid, 100); - console.log('āŒ Should have thrown error for invalid CID'); + const invalidData = new Uint8Array(0); // Empty data + CurioMarket.calculatePieceCID(invalidData); + console.log('āŒ Should have thrown error for invalid data'); } catch (error) { - console.log('āœ… Correctly handled invalid CID:', error.message); + console.log('āœ… Correctly handled invalid data:', error.message); } } catch (error) { diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts index 2b734a0c8..a300bec0a 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/1.ts @@ -10,7 +10,7 @@ // PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 import { getAuthConfigFromEnv, buildAuthHeader, createClient, sanitizeAuthHeader, runPreflightChecks } from './auth'; -import { Mk20Deal, Mk20Products, Mk20PDPV1, Mk20RetrievalV1 } from '../../generated'; +import { CurioMarket } from '../../src'; import { ulid } from 'ulid'; async function sleep(ms: number) { @@ -40,7 +40,7 @@ async function run() { // Create dataset with a fresh identifier (first part of startPDPv1DealForUpload) console.log('šŸ“ Creating PDPv1 dataset...'); const datasetId = ulid(); - const createDeal: Mk20Deal = { + const createDeal: CurioMarket.Deal = { identifier: datasetId, client: config.clientAddr, products: { @@ -51,9 +51,9 @@ async function run() { extraData: [], deleteDataSet: false, deletePiece: false, - } as Mk20PDPV1, - } as Mk20Products, - } as Mk20Deal; + } as CurioMarket.PDPV1, + } as CurioMarket.Products, + } as CurioMarket.Deal; // Submit the dataset creation deal console.log('šŸ“¤ Submitting dataset creation deal...'); diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts index d9dd56c54..81d0879b0 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/2.ts @@ -10,11 +10,10 @@ // PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 import { getAuthConfigFromEnv, buildAuthHeader, createClient, sanitizeAuthHeader, runPreflightChecks } from './auth'; -import { Mk20Deal, Mk20Products, Mk20PDPV1, Mk20RetrievalV1, Mk20DataSource } from '../../generated'; -import { PieceCidUtils } from '../../src'; +import { CurioMarket } from '../../src'; import { ulid } from 'ulid'; -async function sleep(ms: number) { +async function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -68,20 +67,20 @@ async function run(datasetId?: string) { // Compute piece CID console.log('šŸ”— Computing piece CID...'); - const pieceCid = await PieceCidUtils.computePieceCidV2([blob]); + const pieceCid = await CurioMarket.PieceCidUtils.computePieceCidV2([blob]); console.log(` Piece CID: ${pieceCid}`); // Add piece with data under a new identifier (upload id) console.log('šŸ“ Creating add piece deal...'); const uploadId = ulid(); - const addPieceDeal: Mk20Deal = { + const addPieceDeal: CurioMarket.Deal = { identifier: uploadId, client: config.clientAddr, data: { pieceCid: { "/": pieceCid } as object, format: { raw: {} }, sourceHttpPut: {}, - } as Mk20DataSource, + } as CurioMarket.DataSource, products: { pdpV1: { addPiece: true, @@ -90,14 +89,14 @@ async function run(datasetId?: string) { extraData: [], deleteDataSet: false, deletePiece: false, - } as Mk20PDPV1, + } as CurioMarket.PDPV1, retrievalV1: { announcePayload: false, announcePiece: true, indexing: false, - } as Mk20RetrievalV1, - } as Mk20Products, - } as Mk20Deal; + } as CurioMarket.RetrievalV1, + } as CurioMarket.Products, + } as CurioMarket.Deal; // Submit the add piece deal console.log('šŸ“¤ Submitting add piece deal...'); diff --git a/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts b/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts index 7d5a225c5..0fd967863 100644 --- a/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts +++ b/market/mk20/tsclient/examples/unpkg-end-to-end/4.ts @@ -10,7 +10,7 @@ // PDP_CONTRACT=0x4A6867D8537f83c1cEae02dF9Df2E31a6c5A1bb6 import { getAuthConfigFromEnv, buildAuthHeader, createClient } from './auth'; -import { Mk20Deal, Mk20Products, Mk20PDPV1 } from '../../generated'; +import { CurioMarket } from '../../src'; async function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); @@ -47,7 +47,7 @@ async function run(uploadId?: string) { console.log(`šŸ—‘ļø Requesting deletion of upload ${targetUploadId}...`); // Request deletion by updating the deal with delete flags - const deleteDeal: Mk20Deal = { + const deleteDeal: CurioMarket.Deal = { identifier: targetUploadId, client: config.clientAddr, products: { @@ -55,8 +55,8 @@ async function run(uploadId?: string) { deletePiece: true, deleteDataSet: true, recordKeeper: config.recordKeeper, - } as Mk20PDPV1 - } as Mk20Products + } as CurioMarket.PDPV1 + } as CurioMarket.Products }; try { diff --git a/market/mk20/tsclient/src/client.ts b/market/mk20/tsclient/src/client.ts index 56424b085..08b7f5797 100644 --- a/market/mk20/tsclient/src/client.ts +++ b/market/mk20/tsclient/src/client.ts @@ -3,6 +3,7 @@ import { monotonicFactory } from 'ulid'; import { Configuration } from '../generated/runtime'; import { Mk20StartUpload } from '../generated/models/Mk20StartUpload'; import { StreamingPDP } from './streaming'; +import { calculate as calculatePieceCID } from './piece'; const ulid = monotonicFactory(() => Math.random()); export interface MarketClientConfig extends Omit { @@ -11,22 +12,11 @@ export interface MarketClientConfig extends Omit - Piece CID v2 as a string */ @@ -44,236 +34,13 @@ export class PieceCidUtils { offset += uint8Array.length; } - // Compute SHA256 hash (works in browser and Node) - const hashArray = await this.computeSha256(concatenatedData); - - // Create CommP using the exact Go algorithm - const commP = this.newSha2CommP(totalSize, hashArray); - - // Generate piece CID v2 using the exact Go algorithm - const pieceCidV2 = this.pCidV2(commP); - - return pieceCidV2; + // Use the better piece.ts implementation + const pieceCID = calculatePieceCID(concatenatedData); + return pieceCID.toString(); } catch (error) { throw new Error(`Failed to compute piece CID v2: ${error}`); } } - - /** - * Compute SHA-256 digest cross-environment (browser WebCrypto or Node crypto) - */ - private static async computeSha256(data: Uint8Array): Promise { - if (typeof globalThis !== 'undefined' && (globalThis as any).crypto && (globalThis as any).crypto.subtle) { - const h = await (globalThis as any).crypto.subtle.digest('SHA-256', data); - return new Uint8Array(h); - } - try { - const nodeCrypto = await import('crypto'); - const hasher = nodeCrypto.createHash('sha256'); - hasher.update(Buffer.from(data)); - return new Uint8Array(hasher.digest()); - } catch { - throw new Error('No available crypto implementation to compute SHA-256 digest in this environment'); - } - } - - /** - * NewSha2CommP - exact port of Go function - * @param payloadSize - Size of the payload in bytes - * @param digest - 32-byte SHA256 digest - * @returns CommP object - */ - private static newSha2CommP(payloadSize: number, digest: Uint8Array): any { - if (digest.length !== this.NODE_SIZE) { - throw new Error(`digest size must be 32, got ${digest.length}`); - } - - let psz = payloadSize; - - // always 4 nodes long - if (psz < 127) { - psz = 127; - } - - // fr32 expansion, count 127 blocks, rounded up - const boxSize = Math.ceil((psz + 126) / 127) * 128; - - // hardcoded for now - const hashType = 1; - const treeHeight = this.calculateTreeHeight(boxSize); - const payloadPadding = ((1 << (treeHeight - 2)) * 127) - payloadSize; - - return { - hashType, - digest, - treeHeight, - payloadPadding - }; - } - - /** - * Calculate tree height using the exact Go algorithm - * @param boxSize - The box size after fr32 expansion - * @returns Tree height - */ - private static calculateTreeHeight(boxSize: number): number { - // 63 - bits.LeadingZeros64(boxSize) - nodeLog2Size - let leadingZeros = 0; - let temp = boxSize; - while (temp > 0) { - temp = temp >>> 1; - leadingZeros++; - } - leadingZeros = 64 - leadingZeros; - - let treeHeight = 63 - leadingZeros - this.NODE_LOG2_SIZE; - - // if bits.OnesCount64(boxSize) != 1 { treeHeight++ } - if (this.countOnes(boxSize) !== 1) { - treeHeight++; - } - - return treeHeight; - } - - /** - * Count the number of 1 bits in a 64-bit number - * @param n - 64-bit number - * @returns Number of 1 bits - */ - private static countOnes(n: number): number { - let count = 0; - while (n > 0) { - count += n & 1; - n = n >>> 1; - } - return count; - } - - /** - * PCidV2 - exact port of Go function - * @param commP - CommP object - * @returns Piece CID v2 string - */ - private static pCidV2(commP: any): string { - // The Go piece CID v2 format uses a specific prefix structure - // From Go: pCidV2Pref: "\x01" + "\x55" + "\x91" + "\x20" - // This creates: [0x01, 0x55, 0x91, 0x20] = CID v1 + raw codec + multihash length + multihash code - - // Create the complete piece CID v2 structure - // From Go: pCidV2Pref: "\x01" + "\x55" + "\x91" + "\x20" - // This creates: [0x01, 0x55, 0x91, 0x20] = CID v1 + raw codec + multihash length + multihash code - // But the actual piece CID v2 format needs to include the multihash code 0x1011 - const prefix = new Uint8Array([0x01, 0x55, 0x91, 0x20]); // Exact match with Go pCidV2Pref - - // Calculate varint size for payload padding - const ps = this.varintSize(commP.payloadPadding); - - // Create buffer with exact size calculation from Go - const bufSize = prefix.length + 1 + ps + 1 + this.NODE_SIZE; - const buf = new Uint8Array(bufSize); - - let n = 0; - - // Copy prefix - n += this.copyBytes(buf, n, prefix); - - // Set size byte: ps + 1 + nodeSize - buf[n] = ps + 1 + this.NODE_SIZE; - n++; - - // Put varint for payload padding - n += this.putVarint(buf, n, commP.payloadPadding); - - // Set tree height - buf[n] = commP.treeHeight; - n++; - - // Copy digest - this.copyBytes(buf, n, commP.digest); - - // Convert to base32 CID string - return this.bytesToCidString(buf); - } - - /** - * Calculate varint size for a number - * @param value - Number to encode - * @returns Size in bytes - */ - private static varintSize(value: number): number { - if (value < 0x80) return 1; - if (value < 0x4000) return 2; - if (value < 0x200000) return 3; - if (value < 0x10000000) return 4; - if (value < 0x800000000) return 5; - if (value < 0x40000000000) return 6; - if (value < 0x2000000000000) return 7; - if (value < 0x100000000000000) return 8; - return 9; - } - - /** - * Put varint into buffer - * @param buf - Buffer to write to - * @param offset - Offset in buffer - * @param value - Value to encode - * @returns Number of bytes written - */ - private static putVarint(buf: Uint8Array, offset: number, value: number): number { - let n = 0; - while (value >= 0x80) { - buf[offset + n] = (value & 0x7F) | 0x80; - value = value >>> 7; - n++; - } - buf[offset + n] = value & 0x7F; - return n + 1; - } - - /** - * Copy bytes from source to destination - * @param dest - Destination buffer - * @param destOffset - Destination offset - * @param source - Source buffer - * @returns Number of bytes copied - */ - private static copyBytes(dest: Uint8Array, destOffset: number, source: Uint8Array): number { - dest.set(source, destOffset); - return source.length; - } - - /** - * Convert bytes to CID string - * @param bytes - Bytes to convert - * @returns CID string - */ - private static bytesToCidString(bytes: Uint8Array): string { - // This is a simplified conversion - in practice you'd use a proper CID library - // For now, we'll create a base32-like representation - const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; - let result = ''; - let value = 0; - let bits = 0; - - for (let i = 0; i < bytes.length; i++) { - value = (value << 8) | bytes[i]; - bits += 8; - - while (bits >= 5) { - result += base32Chars[(value >>> (bits - 5)) & 31]; - bits -= 5; - } - } - - if (bits > 0) { - result += base32Chars[(value << (5 - bits)) & 31]; - } - - // Add the "b" prefix to match Go's piece CID v2 format - // Go generates: bafkzcibd6adqm6c3a5i7ylct3qkkjtr5qahgt3444eaj5mzhzt2frl7atqscyjwj - return `b${result}`; - } } export class MarketClient { diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts index 7347ffdd6..10115cbd5 100644 --- a/market/mk20/tsclient/src/index.ts +++ b/market/mk20/tsclient/src/index.ts @@ -19,16 +19,17 @@ export type { // Export the main client class export { DefaultApi as MarketClient } from '../generated'; -// Export the custom client wrapper -export { MarketClient as Client } from './client'; +// Export the custom client wrapper and utilities +export { MarketClient as Client, MarketClient, PieceCidUtils } from './client'; export type { MarketClientConfig } from './client'; - -// Export piece CID utilities -export { PieceCidUtils } from './client'; export { StreamingPDP } from './streaming'; export { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; export type { AuthSigner } from './auth'; +// Export piece CID utilities from piece.ts +export { calculate as calculatePieceCID, asPieceCID, asLegacyPieceCID, createPieceCIDStream } from './piece'; +export type { PieceCID, LegacyPieceCID } from './piece'; + // Re-export configuration types export type { Configuration } from '../generated'; @@ -40,3 +41,38 @@ export type { Mk20UploadStatus as UploadStatus, Mk20UploadStatusCode as UploadStatusCode } from '../generated'; + +// Top-level export that encompasses all exports with nice names +export namespace CurioMarket { + // Re-export types with nice names + export type { + Mk20Deal as Deal, + Mk20DataSource as DataSource, + Mk20Products as Products, + Mk20DDOV1 as DDOV1, + Mk20PDPV1 as PDPV1, + Mk20RetrievalV1 as RetrievalV1, + Mk20DealProductStatusResponse as DealProductStatusResponse, + Mk20SupportedContracts as SupportedContracts, + Mk20SupportedProducts as SupportedProducts, + Mk20SupportedDataSources as SupportedDataSources, + Mk20DealCode as DealCode, + Mk20StartUpload as StartUpload, + Mk20UploadCode as UploadCode, + Mk20UploadStartCode as UploadStartCode, + Mk20UploadStatus as UploadStatus, + Mk20UploadStatusCode as UploadStatusCode, + Configuration + } from '../generated'; + + export type { MarketClientConfig } from './client'; + export type { AuthSigner } from './auth'; + export type { PieceCID, LegacyPieceCID } from './piece'; + + // Re-export classes and utilities + export { DefaultApi as MarketClient } from '../generated'; + export { MarketClient as Client, PieceCidUtils } from './client'; + export { StreamingPDP } from './streaming'; + export { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; + export { calculate as calculatePieceCID, asPieceCID, asLegacyPieceCID, createPieceCIDStream } from './piece'; +} diff --git a/market/mk20/tsclient/src/piece.ts b/market/mk20/tsclient/src/piece.ts new file mode 100644 index 000000000..a0ecb238b --- /dev/null +++ b/market/mk20/tsclient/src/piece.ts @@ -0,0 +1,206 @@ +/** + * PieceCID (Piece Commitment CID) utilities + * + * Helper functions for working with Filecoin Piece CIDs + */ + +import type { LegacyPieceLink as LegacyPieceCIDType, PieceLink as PieceCIDType } from '@web3-storage/data-segment' +import * as Hasher from '@web3-storage/data-segment/multihash' +import { CID } from 'multiformats/cid' +import * as Raw from 'multiformats/codecs/raw' +import * as Digest from 'multiformats/hashes/digest' +import * as Link from 'multiformats/link' + +const FIL_COMMITMENT_UNSEALED = 0xf101 +const SHA2_256_TRUNC254_PADDED = 0x1012 + +/** + * PieceCID - A constrained CID type for Piece Commitments. + * This is implemented as a Link type which is made concrete by a CID. A + * PieceCID uses the raw codec (0x55) and the fr32-sha256-trunc254-padbintree + * multihash function (0x1011) which encodes the base content length (as + * padding) of the original piece, and the height of the merkle tree used to + * hash it. + * + * See https://github.com/filecoin-project/FIPs/blob/master/FRCs/frc-0069.md + * for more information. + */ +export type PieceCID = PieceCIDType + +/** + * LegacyPieceCID - A constrained CID type for Legacy Piece Commitments. + * This is implemented as a Link type which is made concrete by a CID. A + * LegacyPieceCID uses the fil-commitment-unsealed codec (0xf101) and the + * sha2-256-trunc254-padded (0x1012) multihash function. + * This 32 bytes of the hash digest in a LegacyPieceCID is the same as the + * equivalent PieceCID, but a LegacyPieceCID does not encode the length or + * tree height of the original raw piece. A PieceCID can be converted to a + * LegacyPieceCID, but not vice versa. + * LegacyPieceCID is commonly known as "CommP" or simply "Piece Commitment" + * in Filecoin. + */ +export type LegacyPieceCID = LegacyPieceCIDType + +/** + * Parse a PieceCID string into a CID and validate it + * @param pieceCidString - The PieceCID as a string (base32 or other multibase encoding) + * @returns The parsed and validated PieceCID CID or null if invalid + */ +function parsePieceCID(pieceCidString: string): PieceCID | null { + try { + const cid = CID.parse(pieceCidString) + if (isValidPieceCID(cid)) { + return cid as PieceCID + } + } catch { + // ignore error + } + return null +} + +/** + * Parse a LegacyPieceCID string into a CID and validate it + * @param pieceCidString - The LegacyPieceCID as a string (base32 or other multibase encoding) + * @returns The parsed and validated LegacyPieceCID CID or null if invalid + */ +function parseLegacyPieceCID(pieceCidString: string): LegacyPieceCID | null { + try { + const cid = CID.parse(pieceCidString) + if (isValidLegacyPieceCID(cid)) { + return cid as LegacyPieceCID + } + } catch { + // ignore error + } + return null +} + +/** + * Check if a CID is a valid PieceCID + * @param cid - The CID to check + * @returns True if it's a valid PieceCID + */ +function isValidPieceCID(cid: PieceCID | CID): cid is PieceCID { + return cid.code === Raw.code && cid.multihash.code === Hasher.code +} + +/** + * Check if a CID is a valid LegacyPieceCID + * @param cid - The CID to check + * @returns True if it's a valid LegacyPieceCID + */ +function isValidLegacyPieceCID(cid: LegacyPieceCID | CID): cid is LegacyPieceCID { + return cid.code === FIL_COMMITMENT_UNSEALED && cid.multihash.code === SHA2_256_TRUNC254_PADDED +} + +/** + * Convert a PieceCID input (string or CID) to a validated CID + * This is the main function to use when accepting PieceCID inputs + * @param pieceCidInput - PieceCID as either a CID object or string + * @returns The validated PieceCID CID or null if not a valid PieceCID + */ +export function asPieceCID(pieceCidInput: PieceCID | CID | string): PieceCID | null { + if (typeof pieceCidInput === 'string') { + return parsePieceCID(pieceCidInput) + } + + if (typeof pieceCidInput === 'object' && CID.asCID(pieceCidInput as CID) !== null) { + // It's already a CID, validate it + if (isValidPieceCID(pieceCidInput as CID)) { + return pieceCidInput as PieceCID + } + } + + // Nope + return null +} + +/** + * Convert a LegacyPieceCID input (string or CID) to a validated CID + * This function can be used to parse a LegacyPieceCID (CommPv1) or to downgrade a PieceCID + * (CommPv2) to a LegacyPieceCID. + * @param pieceCidInput - LegacyPieceCID as either a CID object or string + * @returns The validated LegacyPieceCID CID or null if not a valid LegacyPieceCID + */ +export function asLegacyPieceCID(pieceCidInput: PieceCID | LegacyPieceCID | CID | string): LegacyPieceCID | null { + const pieceCid = asPieceCID(pieceCidInput as CID | string) + if (pieceCid != null) { + // downgrade to LegacyPieceCID + const digest = Digest.create(SHA2_256_TRUNC254_PADDED, pieceCid.multihash.digest.subarray(-32)) + return Link.create(FIL_COMMITMENT_UNSEALED, digest) as LegacyPieceCID + } + + if (typeof pieceCidInput === 'string') { + return parseLegacyPieceCID(pieceCidInput) + } + + if (typeof pieceCidInput === 'object' && CID.asCID(pieceCidInput as CID) !== null) { + // It's already a CID, validate it + if (isValidLegacyPieceCID(pieceCidInput as CID)) { + return pieceCidInput as LegacyPieceCID + } + } + + // Nope + return null +} + +/** + * Calculate the PieceCID (Piece Commitment) for a given data blob + * @param data - The binary data to calculate the PieceCID for + * @returns The calculated PieceCID CID + */ +export function calculate(data: Uint8Array): PieceCID { + // TODO: consider https://github.com/storacha/fr32-sha2-256-trunc254-padded-binary-tree-multihash + // for more efficient PieceCID calculation in WASM + const hasher = Hasher.create() + // We'll get slightly better performance by writing in chunks to let the + // hasher do its work incrementally + const chunkSize = 2048 + for (let i = 0; i < data.length; i += chunkSize) { + hasher.write(data.subarray(i, i + chunkSize)) + } + const digest = hasher.digest() + return Link.create(Raw.code, digest) +} + +/** + * Create a TransformStream that calculates PieceCID while streaming data through it + * This allows calculating PieceCID without buffering the entire data in memory + * + * @returns An object with the TransformStream and a getPieceCID function to retrieve the result + */ +export function createPieceCIDStream(): { + stream: TransformStream + getPieceCID: () => PieceCID | null +} { + const hasher = Hasher.create() + let finished = false + let pieceCid: PieceCID | null = null + + const stream = new TransformStream({ + transform(chunk: Uint8Array, controller: TransformStreamDefaultController) { + // Write chunk to hasher + hasher.write(chunk) + // Pass chunk through unchanged + controller.enqueue(chunk) + }, + + flush() { + // Calculate final PieceCID when stream ends + const digest = hasher.digest() + pieceCid = Link.create(Raw.code, digest) + finished = true + }, + }) + + return { + stream, + getPieceCID: () => { + if (!finished) { + return null + } + return pieceCid + }, + } +} diff --git a/market/mk20/tsclient/src/streaming.ts b/market/mk20/tsclient/src/streaming.ts index eba78a191..005cf6a3e 100644 --- a/market/mk20/tsclient/src/streaming.ts +++ b/market/mk20/tsclient/src/streaming.ts @@ -1,97 +1,9 @@ import { MarketClient as Client } from './client'; import type { Mk20Deal as Deal, Mk20Products as Products, Mk20PDPV1 as PDPV1, Mk20RetrievalV1 as RetrievalV1, Mk20DataSource, Mk20PieceDataFormat } from '../generated'; import { ulid } from 'ulid'; +import { createPieceCIDStream, type PieceCID } from './piece'; -namespace StreamingCommP { - const NODE_SIZE = 32; - const NODE_LOG2_SIZE = 5; - - function calculateTreeHeight(boxSize: number): number { - let leadingZeros = 0; - let temp = boxSize; - while (temp > 0) { - temp = temp >>> 1; - leadingZeros++; - } - leadingZeros = 64 - leadingZeros; - let treeHeight = 63 - leadingZeros - NODE_LOG2_SIZE; - if (countOnes(boxSize) !== 1) treeHeight++; - return treeHeight; - } - - function countOnes(n: number): number { - let count = 0; - while (n > 0) { - count += n & 1; - n = n >>> 1; - } - return count; - } - - function varintSize(value: number): number { - if (value < 0x80) return 1; - if (value < 0x4000) return 2; - if (value < 0x200000) return 3; - if (value < 0x10000000) return 4; - if (value < 0x800000000) return 5; - if (value < 0x40000000000) return 6; - if (value < 0x2000000000000) return 7; - if (value < 0x100000000000000) return 8; - return 9; - } - - function putVarint(buf: Uint8Array, offset: number, value: number): number { - let n = 0; - while (value >= 0x80) { - buf[offset + n] = (value & 0x7f) | 0x80; - value = value >>> 7; - n++; - } - buf[offset + n] = value & 0x7f; - return n + 1; - } - - function bytesToCidString(bytes: Uint8Array): string { - const base32Chars = 'abcdefghijklmnopqrstuvwxyz234567'; - let result = ''; - let value = 0; - let bits = 0; - for (let i = 0; i < bytes.length; i++) { - value = (value << 8) | bytes[i]; - bits += 8; - while (bits >= 5) { - result += base32Chars[(value >>> (bits - 5)) & 31]; - bits -= 5; - } - } - if (bits > 0) { - result += base32Chars[(value << (5 - bits)) & 31]; - } - return `b${result}`; - } - - export function pieceCidV2FromDigest(payloadSize: number, digest: Uint8Array): string { - let psz = payloadSize; - if (psz < 127) psz = 127; - const boxSize = Math.ceil((psz + 126) / 127) * 128; - const treeHeight = calculateTreeHeight(boxSize); - const payloadPadding = ((1 << (treeHeight - 2)) * 127) - payloadSize; - - const prefix = new Uint8Array([0x01, 0x55, 0x91, 0x20]); - const ps = varintSize(payloadPadding); - const bufSize = prefix.length + 1 + ps + 1 + NODE_SIZE; - const buf = new Uint8Array(bufSize); - - let n = 0; - buf.set(prefix, n); n += prefix.length; - buf[n] = ps + 1 + NODE_SIZE; n++; - n += putVarint(buf, n, payloadPadding); - buf[n] = treeHeight; n++; - buf.set(digest, n); - - return bytesToCidString(buf); - } -} +// Removed old custom implementation - now using piece.ts streaming functions /** * StreamingPDP provides a streaming workflow to create a deal without a data section, @@ -102,7 +14,6 @@ export class StreamingPDP { private id: string; private identifierBytes: number[]; private totalSize = 0; - private hashBuffers: Uint8Array[] = []; private deal: Deal | undefined; private clientAddr: string; private providerAddr: string; @@ -112,6 +23,8 @@ export class StreamingPDP { private nextChunkNum = 0; private uploadedBytes = 0; private totalChunks = 0; + private pieceCIDStream: { stream: TransformStream; getPieceCID: () => PieceCID | null }; + private writer: WritableStreamDefaultWriter; /** * @param client - Market client instance @@ -130,6 +43,10 @@ export class StreamingPDP { this.id = ulid(); this.identifierBytes = Array.from(this.id).map(c => c.charCodeAt(0)).slice(0, 16); while (this.identifierBytes.length < 16) this.identifierBytes.push(0); + + // Initialize streaming piece CID computation + this.pieceCIDStream = createPieceCIDStream(); + this.writer = this.pieceCIDStream.stream.writable.getWriter(); } /** @@ -194,11 +111,12 @@ export class StreamingPDP { * and buffers any remainder until the next write or commit. * @param chunk - Data bytes to write */ - write(chunk: Uint8Array | Buffer): void { + async write(chunk: Uint8Array | Buffer): Promise { const u8 = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); this.totalSize += u8.length; - // Cross-env hashing fallback: store chunks for hashing at commit using WebCrypto or Node crypto - this.hashBuffers.push(u8); + + // Write to streaming piece CID computation + await this.writer.write(u8); let idx = 0; if (this.buffer.length > 0) { @@ -208,7 +126,7 @@ export class StreamingPDP { idx += take; if (this.buffer.length === this.chunkSize) { const toSend = this.buffer.slice(0, this.chunkSize); - void this.uploadChunkNow(toSend); + await this.uploadChunkNow(toSend); this.buffer = []; } } @@ -216,7 +134,7 @@ export class StreamingPDP { while (u8.length - idx >= this.chunkSize) { const sub = u8.subarray(idx, idx + this.chunkSize); const toSend = Array.from(sub); - void this.uploadChunkNow(toSend); + await this.uploadChunkNow(toSend); idx += this.chunkSize; } @@ -236,8 +154,13 @@ export class StreamingPDP { this.buffer = []; } - const digest = await this.computeDigest(); - const pieceCid = StreamingCommP.pieceCidV2FromDigest(this.totalSize, digest); + // Close the writer and get the piece CID + await this.writer.close(); + const pieceCID = this.pieceCIDStream.getPieceCID(); + if (!pieceCID) { + throw new Error('Failed to compute piece CID from stream'); + } + const pieceCid = pieceCID.toString(); const dataSource: Mk20DataSource = { pieceCid: { "/": pieceCid } as { [key: string]: string; }, @@ -268,33 +191,16 @@ export class StreamingPDP { } /** - * Compute SHA-256 digest of all streamed bytes using WebCrypto in browsers - * and Node crypto as a fallback in Node environments. + * Clean up resources. Call this when done with the StreamingPDP instance. */ - private async computeDigest(): Promise { - const total = this.hashBuffers.reduce((n, b) => n + b.length, 0); - const all = new Uint8Array(total); - let offset = 0; - for (const b of this.hashBuffers) { - all.set(b, offset); - offset += b.length; - } - - if (typeof globalThis !== 'undefined' && (globalThis as any).crypto && (globalThis as any).crypto.subtle) { - const h = await (globalThis as any).crypto.subtle.digest('SHA-256', all); - return new Uint8Array(h); - } - + async cleanup(): Promise { try { - const nodeCrypto = await import('crypto'); - const hasher = nodeCrypto.createHash('sha256'); - hasher.update(Buffer.from(all)); - return new Uint8Array(hasher.digest()); - } catch { - // Last resort: simple JS fallback (not streaming) using built-in SubtleCrypto if available, else throw - throw new Error('No available crypto implementation to compute SHA-256 digest in this environment'); + await this.writer.close(); + } catch (error) { + // Writer might already be closed } } + } From b91caa510723f794cb095c28b3f7cb01d8d4c8a7 Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Mon, 15 Sep 2025 16:31:52 -0500 Subject: [PATCH 53/55] update ver --- market/mk20/tsclient/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 1d7cf687a..9b96e3e2c 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -1,6 +1,6 @@ { "name": "@curiostorage/market-client", - "version": "0.4.1", + "version": "0.4.2", "description": "TypeScript API client for Curio storage market. You probably want the Synapse SDK instead.", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", From c3f52bfeadd843f012a235738c5adf9cb470d7cc Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Wed, 17 Sep 2025 09:31:46 -0500 Subject: [PATCH 54/55] rename --- market/mk20/tsclient/src/index.ts | 92 ++++++++--------------- market/mk20/tsclient/test-curio-market.ts | 22 ++++++ 2 files changed, 54 insertions(+), 60 deletions(-) create mode 100644 market/mk20/tsclient/test-curio-market.ts diff --git a/market/mk20/tsclient/src/index.ts b/market/mk20/tsclient/src/index.ts index 10115cbd5..9cb28212d 100644 --- a/market/mk20/tsclient/src/index.ts +++ b/market/mk20/tsclient/src/index.ts @@ -1,7 +1,30 @@ // Export the generated client and types export * from '../generated'; -// Re-export commonly used types for convenience +// Import everything we need for the CurioMarket object +import { DefaultApi as MarketClient } from '../generated'; +import { MarketClient as Client, PieceCidUtils } from './client'; +import { StreamingPDP } from './streaming'; +import { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; +import { calculate as calculatePieceCID, asPieceCID, asLegacyPieceCID, createPieceCIDStream } from './piece'; + +// Top-level export that encompasses all exports with nice names +export const CurioMarket = { + // Classes and utilities + MarketClient, + Client, + PieceCidUtils, + StreamingPDP, + AuthUtils, + Ed25519KeypairSigner, + Secp256k1AddressSigner, + calculatePieceCID, + asPieceCID, + asLegacyPieceCID, + createPieceCIDStream, +} as const; + +// Export types with nice names export type { Mk20Deal as Deal, Mk20DataSource as DataSource, @@ -13,66 +36,15 @@ export type { Mk20SupportedContracts as SupportedContracts, Mk20SupportedProducts as SupportedProducts, Mk20SupportedDataSources as SupportedDataSources, - Mk20DealCode as DealCode + Mk20DealCode as DealCode, + Mk20StartUpload as StartUpload, + Mk20UploadCode as UploadCode, + Mk20UploadStartCode as UploadStartCode, + Mk20UploadStatus as UploadStatus, + Mk20UploadStatusCode as UploadStatusCode, + Configuration } from '../generated'; -// Export the main client class -export { DefaultApi as MarketClient } from '../generated'; - -// Export the custom client wrapper and utilities -export { MarketClient as Client, MarketClient, PieceCidUtils } from './client'; export type { MarketClientConfig } from './client'; -export { StreamingPDP } from './streaming'; -export { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; export type { AuthSigner } from './auth'; - -// Export piece CID utilities from piece.ts -export { calculate as calculatePieceCID, asPieceCID, asLegacyPieceCID, createPieceCIDStream } from './piece'; -export type { PieceCID, LegacyPieceCID } from './piece'; - -// Re-export configuration types -export type { Configuration } from '../generated'; - -// Re-export upload-related types for convenience -export type { - Mk20StartUpload as StartUpload, - Mk20UploadCode as UploadCode, - Mk20UploadStartCode as UploadStartCode, - Mk20UploadStatus as UploadStatus, - Mk20UploadStatusCode as UploadStatusCode -} from '../generated'; - -// Top-level export that encompasses all exports with nice names -export namespace CurioMarket { - // Re-export types with nice names - export type { - Mk20Deal as Deal, - Mk20DataSource as DataSource, - Mk20Products as Products, - Mk20DDOV1 as DDOV1, - Mk20PDPV1 as PDPV1, - Mk20RetrievalV1 as RetrievalV1, - Mk20DealProductStatusResponse as DealProductStatusResponse, - Mk20SupportedContracts as SupportedContracts, - Mk20SupportedProducts as SupportedProducts, - Mk20SupportedDataSources as SupportedDataSources, - Mk20DealCode as DealCode, - Mk20StartUpload as StartUpload, - Mk20UploadCode as UploadCode, - Mk20UploadStartCode as UploadStartCode, - Mk20UploadStatus as UploadStatus, - Mk20UploadStatusCode as UploadStatusCode, - Configuration - } from '../generated'; - - export type { MarketClientConfig } from './client'; - export type { AuthSigner } from './auth'; - export type { PieceCID, LegacyPieceCID } from './piece'; - - // Re-export classes and utilities - export { DefaultApi as MarketClient } from '../generated'; - export { MarketClient as Client, PieceCidUtils } from './client'; - export { StreamingPDP } from './streaming'; - export { AuthUtils, Ed25519KeypairSigner, Secp256k1AddressSigner } from './auth'; - export { calculate as calculatePieceCID, asPieceCID, asLegacyPieceCID, createPieceCIDStream } from './piece'; -} +export type { PieceCID, LegacyPieceCID } from './piece'; \ No newline at end of file diff --git a/market/mk20/tsclient/test-curio-market.ts b/market/mk20/tsclient/test-curio-market.ts new file mode 100644 index 000000000..75b8d50b1 --- /dev/null +++ b/market/mk20/tsclient/test-curio-market.ts @@ -0,0 +1,22 @@ +// Test script to verify CurioMarket object structure +import { CurioMarket } from './src'; + +console.log('CurioMarket object structure:'); +console.log('Available properties:', Object.keys(CurioMarket)); + +console.log('\nClasses available:'); +console.log('- MarketClient:', typeof CurioMarket.MarketClient); +console.log('- Client:', typeof CurioMarket.Client); +console.log('- PieceCidUtils:', typeof CurioMarket.PieceCidUtils); +console.log('- StreamingPDP:', typeof CurioMarket.StreamingPDP); +console.log('- AuthUtils:', typeof CurioMarket.AuthUtils); + +console.log('\nFunctions available:'); +console.log('- calculatePieceCID:', typeof CurioMarket.calculatePieceCID); +console.log('- asPieceCID:', typeof CurioMarket.asPieceCID); +console.log('- asLegacyPieceCID:', typeof CurioMarket.asLegacyPieceCID); +console.log('- createPieceCIDStream:', typeof CurioMarket.createPieceCIDStream); + +console.log('\nāœ… CurioMarket object is properly structured!'); + + From 23633d2a0a76cc9c5e8532ba34a244bb49b37f9b Mon Sep 17 00:00:00 2001 From: "Andrew Jackson (Ajax)" Date: Thu, 18 Sep 2025 11:38:40 -0500 Subject: [PATCH 55/55] packaging ts --- market/mk20/tsclient/package-lock.json | 52 ++++++++++++++++++++++++-- market/mk20/tsclient/package.json | 1 + market/mk20/tsclient/tsconfig.json | 4 +- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/market/mk20/tsclient/package-lock.json b/market/mk20/tsclient/package-lock.json index 145a16269..5117d0ccc 100644 --- a/market/mk20/tsclient/package-lock.json +++ b/market/mk20/tsclient/package-lock.json @@ -1,16 +1,17 @@ { - "name": "@curio/market-client", - "version": "1.0.0", + "name": "@curiostorage/market-client", + "version": "0.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@curio/market-client", - "version": "1.0.0", + "name": "@curiostorage/market-client", + "version": "0.4.2", "license": "MIT", "dependencies": { "@glif/filecoin-address": "^4.0.0", "@noble/secp256k1": "^2.1.0", + "@web3-storage/data-segment": "^5.3.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", "tweetnacl": "^1.0.3", @@ -590,6 +591,20 @@ } } }, + "node_modules/@ipld/dag-cbor": { + "version": "9.2.5", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.5.tgz", + "integrity": "sha512-84wSr4jv30biui7endhobYhXBQzQE4c/wdoWlFrKcfiwH+ofaPg8fwsM8okX9cOzkkrsAsNdDyH3ou+kiLquwQ==", + "license": "Apache-2.0 OR MIT", + "dependencies": { + "cborg": "^4.0.0", + "multiformats": "^13.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@isaacs/balanced-match": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", @@ -1514,6 +1529,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@web3-storage/data-segment": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@web3-storage/data-segment/-/data-segment-5.3.0.tgz", + "integrity": "sha512-zFJ4m+pEKqtKatJNsFrk/2lHeFSbkXZ6KKXjBe7/2ayA9wAar7T/unewnOcZrrZTnCWmaxKsXWqdMFy9bXK9dw==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-cbor": "^9.2.1", + "multiformats": "^13.3.0", + "sync-multihash-sha2": "^1.0.0" + } + }, "node_modules/aes-js": { "version": "4.0.0-beta.5", "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-4.0.0-beta.5.tgz", @@ -2044,6 +2070,15 @@ ], "license": "CC-BY-4.0" }, + "node_modules/cborg": { + "version": "4.2.15", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.2.15.tgz", + "integrity": "sha512-T+YVPemWyXcBVQdp0k61lQp2hJniRNmul0lAwTj2DTS/6dI4eCq/MRMucGqqvFqMBfmnD8tJ9aFtPu5dEGAbgw==", + "license": "Apache-2.0", + "bin": { + "cborg": "lib/bin.js" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -5529,6 +5564,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sync-multihash-sha2": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/sync-multihash-sha2/-/sync-multihash-sha2-1.0.0.tgz", + "integrity": "sha512-A5gVpmtKF0ov+/XID0M0QRJqF2QxAsj3x/LlDC8yivzgoYCoWkV+XaZPfVu7Vj1T/hYzYS1tfjwboSbXjqocug==", + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@noble/hashes": "^1.3.1" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", diff --git a/market/mk20/tsclient/package.json b/market/mk20/tsclient/package.json index 9b96e3e2c..6a196cc52 100644 --- a/market/mk20/tsclient/package.json +++ b/market/mk20/tsclient/package.json @@ -35,6 +35,7 @@ "dependencies": { "@glif/filecoin-address": "^4.0.0", "@noble/secp256k1": "^2.1.0", + "@web3-storage/data-segment": "^5.3.0", "isomorphic-fetch": "^3.0.0", "multiformats": "^13.4.0", "tweetnacl": "^1.0.3", diff --git a/market/mk20/tsclient/tsconfig.json b/market/mk20/tsclient/tsconfig.json index 7e9d3aa9b..b710e6cce 100644 --- a/market/mk20/tsclient/tsconfig.json +++ b/market/mk20/tsclient/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "target": "ES2020", - "module": "commonjs", + "module": "NodeNext", "lib": ["ES2020", "DOM"], "types": ["node"], "declaration": true, @@ -12,7 +12,7 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "resolveJsonModule": true, - "moduleResolution": "node", + "moduleResolution": "nodenext", "allowSyntheticDefaultImports": true, "experimentalDecorators": true, "emitDecoratorMetadata": true