Merge branch 'main' into market-migration

Author: LexLuthr

.circleci/config.yml

@@ -279,6 +279,7 @@ workflows:
           name: test-all
           requires:
             - build
-          target: "`go list ./... | grep -v curio/itests`"
           suite: test-all
+          target: "`go list ./... | grep -v curio/itests`"
           get-params: true
+          resource_class: 2xlarge

README.md

@@ -40,12 +40,22 @@ Curio Storage is designed to deliver significant improvements and new features f
 - **Scalability**: No single point of stress, allowing for seamless expansion from a single machine to thousands.
 - **Multi-Miner-ID Support**: Saves hardware resources by enabling multi-proving.
 - **Rich Filtering**: Allows detailed filtering at storage locations by ID, purpose, etc.
+- **Better Hardware Efficiency**: Tasks pack tightly onto any machine that has that work layer enabled.
+- **Debian/Ubuntu APT Repo**: Automatic updates & no building (for the common case).
+- **Simple Migrator**: Switch from Lotus Miner to Curio in minutes.
 
 ## Future Developments
 
-Curio Storage is continuously evolving, with future plans including:
+Since release, Curio's team has added:
 
 - **New Snap Sectors & Unsealing**: Completing the feature set of lotus-miner.
+- **Curio SupraSeal (beta) integration**: 10x more efficient Sealing than through other means (Read docs).
+- **Disk Garbage Collection**: Know you're storing only what you need.
+- **Binary size reduction**: All this functionality is now in a binary under 100MiB.
+
+
+Curio Storage is continuously evolving, with future plans including:
+
 - **Built-In Markets (like Boost) with Automation**: Enhancing market capabilities with automation.
 - **Staking, Resource Sharing, & Deals 2.0**: Introducing advanced features for resource management.
 - **Efficiency Gains**: Achieving significant cost savings for storage providers of all sizes.
@@ -58,7 +68,9 @@ Curio Storage is continuously evolving, with future plans including:
 
 ## Getting Started
 
-To get started with Curio Storage, follow these steps:
+We recommend reading [docs.CurioStorage.org](https://docs.curiostorage.org/) where you may have a pre-built binary for your system.
+
+One way to get started with Curio Storage, follow these steps:
 
 1. **Clone the Repository**:
     ```sh
@@ -89,7 +101,7 @@ Dual-licensed under [MIT](https://github.com/filecoin-project/curio/blob/master/
 
 # Contributing
 
-We welcome contributions from the community! Please see our [Contributing Guide](CONTRIBUTING.md) for more details.
+We welcome contributions from the community! Feel free to open PRs. Reach out to the team on Slack if you need any help.
 
 ---
 

api/api_curio.go

@@ -8,10 +8,11 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 
+	storiface "github.com/filecoin-project/curio/lib/storiface"
+
 	"github.com/filecoin-project/lotus/api"
 	lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 type Curio interface {

api/proxy_gen.go

@@ -328,7 +328,7 @@ func configToDB(d *MigrationData) {
 		return i == 1
 	}
 
-	d.MinerID, err = SaveConfigToLayerMigrateSectors(d.MinerConfigPath, chainApiInfo, shouldErrPrompt)
+	d.MinerID, err = SaveConfigToLayerMigrateSectors(d.DB, d.MinerConfigPath, chainApiInfo, shouldErrPrompt)
 	if err != nil {
 		d.say(notice, "Error saving config to layer: %s. Aborting Migration", err.Error())
 		os.Exit(1)

cmd/curio/guidedsetup/guidedsetup.go

@@ -26,11 +26,11 @@ import (
 	"github.com/filecoin-project/curio/deps"
 	"github.com/filecoin-project/curio/deps/config"
 	"github.com/filecoin-project/curio/harmony/harmonydb"
+	"github.com/filecoin-project/curio/lib/storiface"
 	"github.com/filecoin-project/curio/lib/types/sector"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 //go:generate cbor-gen-for --map-encoding SectorInfo
@@ -68,7 +68,7 @@ func (storageMiner) APIInfoEnvVars() (primary string, fallbacks []string, deprec
 	return "MINER_API_INFO", nil, []string{"STORAGE_API_INFO"}
 }
 
-func SaveConfigToLayerMigrateSectors(minerRepoPath, chainApiInfo string, unmigSectorShouldFail func() bool) (minerAddress address.Address, err error) {
+func SaveConfigToLayerMigrateSectors(db *harmonydb.DB, minerRepoPath, chainApiInfo string, unmigSectorShouldFail func() bool) (minerAddress address.Address, err error) {
 	_, say := SetupLanguage()
 	ctx := context.Background()
 
@@ -103,12 +103,6 @@ func SaveConfigToLayerMigrateSectors(minerRepoPath, chainApiInfo string, unmigSe
 	}
 	smCfg := cfgNode.(*config.StorageMiner)
 
-	db, err := harmonydb.NewFromConfig(smCfg.HarmonyDB)
-	if err != nil {
-		return minerAddress, fmt.Errorf("could not reach the database. Ensure the Miner config toml's HarmonyDB entry"+
-			" is setup to reach Yugabyte correctly: %w", err)
-	}
-
 	var titles []string
 	err = db.Select(ctx, &titles, `SELECT title FROM harmony_config WHERE LENGTH(config) > 0`)
 	if err != nil {

cmd/curio/guidedsetup/shared.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/ipfs/go-datastore"
+	"github.com/samber/lo"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
@@ -29,6 +31,7 @@ var sealCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		sealStartCmd,
 		sealMigrateLMSectorsCmd,
+		sealEventsCmd,
 	},
 }
 
@@ -253,3 +256,108 @@ var sealMigrateLMSectorsCmd = &cli.Command{
 		return nil
 	},
 }
+
+var sealEventsCmd = &cli.Command{
+	Name:  "events",
+	Usage: "List pipeline events",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "Filter events by actor address; lists all if not specified",
+		},
+		&cli.IntFlag{
+			Name:  "sector",
+			Usage: "Filter events by sector number; requires --actor to be specified",
+		},
+		&cli.UintFlag{
+			Name:  "last",
+			Usage: "Limit output to the last N events",
+			Value: 100,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+
+		var actorID uint64
+		var sector abi.SectorNumber
+		if cctx.IsSet("actor") {
+			act, err := address.NewFromString(cctx.String("actor"))
+			if err != nil {
+				return fmt.Errorf("parsing --actor: %w", err)
+			}
+			mid, err := address.IDFromAddress(act)
+			if err != nil {
+				return fmt.Errorf("getting miner id: %w", err)
+			}
+			actorID = mid
+		}
+		if cctx.IsSet("sector") {
+			sector = abi.SectorNumber(cctx.Int("sector"))
+			if !cctx.IsSet("actor") {
+				return fmt.Errorf("the --actor flag is required when using --sector")
+			}
+		}
+
+		ctx := reqcontext.ReqContext(cctx)
+		dep, err := deps.GetDepsCLI(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		var events []struct {
+			ActorID   int       `db:"sp_id"`
+			Sector    int       `db:"sector_number"`
+			ID        int       `db:"id"`
+			TaskID    int       `db:"task_id"`
+			Name      string    `db:"name"`
+			Posted    time.Time `db:"posted"`
+			WorkStart time.Time `db:"work_start"`
+			WorkEnd   time.Time `db:"work_end"`
+			Result    bool      `db:"result"`
+			Err       string    `db:"err"`
+			ByHost    string    `db:"completed_by_host_and_port"`
+		}
+
+		if !cctx.IsSet("actor") {
+			// list for all actors
+			err = dep.DB.Select(ctx, &events, `SELECT s.sp_id, s.sector_number, h.*
+				FROM harmony_task_history h
+         			JOIN sectors_pipeline_events s ON h.id = s.task_history_id
+				ORDER BY h.work_end DESC LIMIT $1;`, cctx.Int("last"))
+		} else if cctx.IsSet("sector") {
+			// list for specific actor and sector
+			err = dep.DB.Select(ctx, &events, `SELECT s.sp_id, s.sector_number, h.*
+				FROM harmony_task_history h
+         			JOIN sectors_pipeline_events s ON h.id = s.task_history_id
+				WHERE s.sp_id = $1 AND s.sector_number = $2 ORDER BY h.work_end DESC LIMIT $3;`, actorID, sector, cctx.Int("last"))
+		} else {
+			fmt.Println(cctx.IsSet("actor"), cctx.IsSet("sector"))
+			// list for specific actor
+			err = dep.DB.Select(ctx, &events, `SELECT s.sp_id, s.sector_number, h.*
+				FROM harmony_task_history h
+         			JOIN sectors_pipeline_events s ON h.id = s.task_history_id
+				WHERE s.sp_id = $1 ORDER BY h.work_end DESC LIMIT $2;`, actorID, cctx.Int("last"))
+		}
+
+		if err != nil {
+			return fmt.Errorf("getting events: %w", err)
+		}
+		fmt.Printf("Total Events: %d\n", len(events))
+
+		fmt.Printf("%-10s %-10s %-18s %-10s %-28s %-28s %-20s %-20s %-10s %-20s \n", "ActorID", "Sector", "Task", "HistoryID", "Posted", "Start", "Took", "Machine", "Status", "Error")
+		for _, e := range events {
+			fmt.Printf("%-10d %-10d %-18s %-10d %-28s %-28s %-20s %-20s %-10s %-20s \n",
+				e.ActorID,
+				e.Sector,
+				e.Name,
+				e.TaskID,
+				e.Posted.Format(time.RFC3339),
+				e.WorkStart.Format(time.RFC3339),
+				e.WorkEnd.Sub(e.WorkStart),
+				e.ByHost,
+				lo.Ternary(e.Result, "Success", "Failure"),
+				e.Err,
+			)
+		}
+		return nil
+	},
+}

cmd/curio/guidedsetup/shared_cbor_gen.go

@@ -33,6 +33,7 @@ import (
 	"github.com/filecoin-project/curio/lib/metrics"
 	"github.com/filecoin-project/curio/lib/paths"
 	"github.com/filecoin-project/curio/lib/repo"
+	storiface "github.com/filecoin-project/curio/lib/storiface"
 	"github.com/filecoin-project/curio/web"
 
 	lapi "github.com/filecoin-project/lotus/api"
@@ -42,7 +43,6 @@ import (
 	"github.com/filecoin-project/lotus/metrics/proxy"
 	"github.com/filecoin-project/lotus/storage/pipeline/piece"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 const metaFile = "sectorstore.json"
@@ -296,7 +296,7 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c
 	eg.Go(srv.ListenAndServe)
 
 	if dependencies.Cfg.Subsystems.EnableWebGui {
-		web, err := web.GetSrv(ctx, dependencies)
+		web, err := web.GetSrv(ctx, dependencies, false)
 		if err != nil {
 			return err
 		}

cmd/curio/pipeline.go

@@ -20,10 +20,10 @@ import (
 
 	"github.com/filecoin-project/curio/cmd/curio/rpc"
 	"github.com/filecoin-project/curio/lib/reqcontext"
+	storiface "github.com/filecoin-project/curio/lib/storiface"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/storage/sealer/fsutil"
-	"github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var storageCmd = &cli.Command{
@@ -438,7 +438,7 @@ var storageFindCmd = &cli.Command{
 		}
 
 		sectorTypes := []storiface.SectorFileType{
-			storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache,
+			storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache, storiface.FTPiece,
 		}
 
 		byId := make(map[storiface.ID]*storedSector)