fix ingest, market tasks

Author: LexLuthr

Makefile

@@ -50,10 +50,11 @@ build/.update-modules:
 # end git modules
 
 ## CUDA Library Path
-CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc)))
-CUDA_LIB_PATH := $(CUDA_PATH)/lib64
-LIBRARY_PATH ?= $(CUDA_LIB_PATH)
-export LIBRARY_PATH
+setup_cuda:
+	$(eval CUDA_PATH := $(shell dirname $$(dirname $$(which nvcc))))
+	$(eval CUDA_LIB_PATH := $(CUDA_PATH)/lib64)
+	export LIBRARY_PATH=$(CUDA_LIB_PATH)
+.PHONY: setup_cuda
 
 ## MAIN BINARIES
 
@@ -81,9 +82,9 @@ BINS+=sptool
 
 ifeq ($(shell uname),Linux)
 
-batchdep: build/.supraseal-install
+batchdep: setup_cuda build/.supraseal-install
 batchdep: $(BUILD_DEPS)
-,PHONY: batchdep
+.PHONY: batchdep
 
 batch: GOFLAGS+=-tags=supraseal
 batch: CGO_LDFLAGS_ALLOW='.*'

deps/config/doc_gen.go

@@ -217,24 +217,20 @@ description: The default curio configuration
   # type: int
   #SyntheticPoRepMaxTasks = 0
 
-<<<<<<< HEAD
-<<<<<<< HEAD
   # Batch Seal
   #
   # type: bool
   #EnableBatchSeal = false
-=======
+
   # EnableDealMarket
   #
   # type: bool
   #EnableDealMarket = false
->>>>>>> 6d3d660 (basic mk12 scafolding)
-=======
+
   # EnableCommP enabled the commP task on te node. CommP is calculated before sending PublishDealMessage for a Mk12 deal
   #
   # type: bool
   #EnableCommP = false
->>>>>>> bd6e6c0 (poller redesign)
 
 
 [Fees]
@@ -416,7 +412,7 @@ description: The default curio configuration
       #InsertConcurrency = 8
 
     [Market.StorageMarketConfig.MK12]
-      # Miners is a list of miner to enable MK12 deals(Boost) for
+      # Miners is a list of miner addresses to enable MK12 deals(Boost) for
       #
       # type: []string
       #Miners = []
@@ -425,27 +421,32 @@ description: The default curio configuration
       # deals to be ready to publish before publishing them all as a batch
       #
       # type: Duration
-      #PublishMsgPeriod = "0s"
+      #PublishMsgPeriod = "30m0s"
 
       # The maximum number of deals to include in a single PublishStorageDeals
       # message
       #
       # type: uint64
-      #MaxDealsPerPublishMsg = 0
+      #MaxDealsPerPublishMsg = 8
 
       # The maximum collateral that the provider will put up against a deal,
       # as a multiplier of the minimum collateral bound
       # The maximum fee to pay when sending the PublishStorageDeals message
       #
       # type: types.FIL
-      #MaxPublishDealsFee = "0 FIL"
+      #MaxPublishDealsFee = "5 FIL"
+
+      # ExpectedPoRepSealDuration is the expected time it would take to seal the deal sector
+      # This will be used to fail the deals which cannot be sealed on time.
+      #
+      # type: Duration
+      #ExpectedPoRepSealDuration = "8h0m0s"
 
-      # ExpectedSealDuration is the expected time it would take to seal the deal sector
+      # ExpectedSnapSealDuration is the expected time it would take to snap the deal sector
       # This will be used to fail the deals which cannot be sealed on time.
-      # Please make sure to update this to shorter duration for snap deals
       #
       # type: Duration
-      #ExpectedSealDuration = "0s"
+      #ExpectedSnapSealDuration = "2h0m0s"
 
       # SkipCommP can be used to skip doing a commP check before PublishDealMessage is sent on chain
       # Warning: If this check is skipped and there is a commP mismatch, all deals in the

documentation/en/configuration/default-curio-configuration.md

@@ -9,8 +9,6 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/filecoin-project/curio/build"
-	"github.com/filecoin-project/curio/deps/config"
 	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
@@ -22,6 +20,8 @@ import (
 	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/filecoin-project/go-state-types/network"
 
+	"github.com/filecoin-project/curio/build"
+	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
 	"github.com/filecoin-project/curio/tasks/seal"
 
@@ -38,6 +38,7 @@ var log = logging.Logger("storage-ingest")
 type Ingester interface {
 	AllocatePieceToSector(ctx context.Context, tx *harmonydb.Tx, maddr address.Address, piece lpiece.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (api.SectorOffset, error)
 	SectorStartSealing(ctx context.Context, maddr address.Address, sector abi.SectorNumber) error
+	GetExpectedSealDuration() abi.ChainEpoch
 }
 
 type PieceIngesterApi interface {
@@ -572,3 +573,7 @@ func (p *PieceIngester) getOpenSectors(tx *harmonydb.Tx, mid int64) ([]*openSect
 
 	return os, nil
 }
+
+func (p *PieceIngester) GetExpectedSealDuration() abi.ChainEpoch {
+	return p.expectedSealDuration
+}

lib/paths/mocks/index.go

@@ -9,8 +9,6 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/filecoin-project/curio/build"
-	"github.com/filecoin-project/curio/deps/config"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -22,6 +20,8 @@ import (
 	verifreg13 "github.com/filecoin-project/go-state-types/builtin/v13/verifreg"
 	"github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
 
+	"github.com/filecoin-project/curio/build"
+	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
 
 	"github.com/filecoin-project/lotus/api"
@@ -693,3 +693,7 @@ func (p *PieceIngesterSnap) getOpenSectors(tx *harmonydb.Tx, mid int64) ([]*open
 
 	return os, nil
 }
+
+func (p *PieceIngesterSnap) GetExpectedSealDuration() abi.ChainEpoch {
+	return p.expectedSnapDuration
+}

lib/paths/mocks/pf.go

@@ -5,11 +5,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"net/url"
 	"strconv"
-	"time"
 
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
@@ -20,7 +18,6 @@ import (
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/go-state-types/abi"
 
-	"github.com/filecoin-project/curio/build"
 	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
 	"github.com/filecoin-project/curio/harmony/harmonytask"
@@ -73,7 +70,7 @@ func (c *CommpTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
 	}
 	piece := pieces[0]
 
-	expired, err := checkExpiry(ctx, c.db, c.api, piece.UUID, c.cfg.ExpectedSealDuration)
+	expired, err := checkExpiry(ctx, c.db, c.api, piece.UUID, c.sm.pin.GetExpectedSealDuration())
 	if err != nil {
 		return false, xerrors.Errorf("deal %s expired: %w", piece.UUID, err)
 	}
@@ -316,7 +313,7 @@ type headAPI interface {
 	ChainHead(context.Context) (*types.TipSet, error)
 }
 
-func checkExpiry(ctx context.Context, db *harmonydb.DB, api headAPI, deal string, buffer config.Duration) (bool, error) {
+func checkExpiry(ctx context.Context, db *harmonydb.DB, api headAPI, deal string, sealDuration abi.ChainEpoch) (bool, error) {
 	var starts []struct {
 		StartEpoch int64 `db:"start_epoch"`
 	}
@@ -333,9 +330,7 @@ func checkExpiry(ctx context.Context, db *harmonydb.DB, api headAPI, deal string
 		return false, err
 	}
 
-	buff := int64(math.Floor(time.Duration(buffer).Seconds() / float64(build.BlockDelaySecs)))
-
-	if head.Height()+abi.ChainEpoch(buff) > startEPoch {
+	if head.Height()+sealDuration > startEPoch {
 		err = failDeal(ctx, db, deal, true, fmt.Sprintf("deal proposal must be proven on chain by deal proposal start epoch %d, but it has expired: current chain height: %d",
 			startEPoch, head.Height()))
 		return true, err

lib/paths/mocks/store.go

@@ -88,7 +88,7 @@ func (f *FindDealTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (do
 	}
 	bd := bdeals[0]
 
-	expired, err := checkExpiry(ctx, f.db, f.api, bd.UUID, f.cfg.ExpectedSealDuration)
+	expired, err := checkExpiry(ctx, f.db, f.api, bd.UUID, f.sm.pin.GetExpectedSealDuration())
 	if err != nil {
 		return false, xerrors.Errorf("deal %s expired: %w", bd.UUID, err)
 	}