feat: market 2.0 (#508)

* initial draft of market 2.0

* PUT endpoint for deals

* add some type validation tests

* info page and pdp router changes

* client, commv2, ui

* mk20 GC

* aggregate deals

* offline deal and put deals

* finish UI

* UI, reindex, retrievals

* update pdp client service url

* basic docs, new api, UI changes

* chunk upload

* fix actor info page

* add some PDP metrics

* use piecepark for upload

* pcid2, pdpv1, retrievalv1

* PDP pipeline

* openAPI, auth, streamline types and methods

* pdp prove test

* deleteRoot, deleteProofSet

* serial upload

* working PDP pipeline, indexing, ipni

* rename PDP

* lotus version

* CommPV2 integration with PDP contracts

* piece cleanup, string client

* IPDPProvingSchedule changes

* fix swagger definitions

* switch to camelCase for json

* undo m20 ddo download

* undo camelCase

* fix test

* fix status, add status command

* fix swagger types, UI fixes

* fix pdp_ipni, ipni_ui

* leftover PDP rename

* fix swagger, ppinit

* refactor cleanup, fix proving

* fix tests

* fix savecache and proving

* make gen

* fix cleanup, drop pdp cache

* FS registry

* fix translation and make gen

Author: LexLuthr

.github/workflows/ci.yml

@@ -179,6 +179,8 @@ jobs:
             target: "./itests/harmonydb_test.go"
           - name: test-itest-alertnow
             target: "./itests/alertnow_test.go"
+          - name: test-itest-pdp-prove
+            target: "./itests/pdp_prove_test.go"
     steps:
       - uses: actions/checkout@v4
 
@@ -311,6 +313,10 @@ jobs:
         run: go install github.com/hannahhoward/cbor-gen-for
         shell: bash
 
+      - name: Install swag cli
+        run: go install github.com/swaggo/swag/cmd/[email protected]
+        shell: bash
+
 #      - name: Install gotext
 #        run: go install golang.org/x/text/cmd/gotext
 #        shell: bash

Dockerfile

@@ -14,6 +14,11 @@ ENV RUSTUP_HOME=/usr/local/rustup \
     PATH=/usr/local/cargo/bin:$PATH \
     RUST_VERSION=1.63.0
 
+COPY ./ /opt/curio
+WORKDIR /opt/curio
+RUN git submodule update --init
+RUN go mod download
+
 RUN set -eux; \
     dpkgArch="$(dpkg --print-architecture)"; \
     case "${dpkgArch##*-}" in \
@@ -32,9 +37,6 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
-COPY ./ /opt/curio
-WORKDIR /opt/curio
-
 ### make configurable filecoin-ffi build
 ARG FFI_BUILD_FROM_SOURCE=0
 ENV FFI_BUILD_FROM_SOURCE=${FFI_BUILD_FROM_SOURCE}
@@ -56,13 +58,26 @@ RUN go install github.com/ipld/go-car/cmd/car@latest \
 RUN go install github.com/LexLuthr/piece-server@latest \
  && cp $GOPATH/bin/piece-server /usr/local/bin/
 
-RUN go install github.com/ipni/storetheindex@v0.8.38 \
+RUN go install github.com/ipni/storetheindex@latest \
  && cp $GOPATH/bin/storetheindex /usr/local/bin/
 
+RUN go install github.com/ethereum/go-ethereum/cmd/geth@latest \
+ && cp $GOPATH/bin/geth /usr/local/bin/
+
 #####################################
 FROM ubuntu:22.04 AS curio-all-in-one
 
-RUN apt-get update && apt-get install -y dnsutils vim curl aria2 jq
+RUN apt-get update && apt-get install -y dnsutils vim curl aria2 jq git wget nodejs npm
+
+# Install Foundry
+RUN curl -L https://foundry.paradigm.xyz | bash \
+    && bash -c ". ~/.foundry/bin/foundryup"
+
+# Make sure foundry binaries are available in PATH
+ENV PATH="/root/.foundry/bin:${PATH}"
+
+# Verify installation
+RUN forge --version && cast --version && anvil --version
 
 # Copy libraries and binaries from curio-builder
 COPY --from=curio-builder /etc/ssl/certs /etc/ssl/certs
@@ -98,6 +113,7 @@ COPY --from=curio-builder /opt/curio/sptool /usr/local/bin/
 COPY --from=piece-server-builder /usr/local/bin/piece-server /usr/local/bin/
 COPY --from=piece-server-builder /usr/local/bin/car /usr/local/bin/
 COPY --from=piece-server-builder /usr/local/bin/storetheindex /usr/local/bin/
+COPY --from=piece-server-builder /usr/local/bin/geth /usr/local/bin/
 
 # Set up directories and permissions
 RUN mkdir /var/tmp/filecoin-proof-parameters \

Makefile

@@ -266,7 +266,11 @@ go-generate:
 gen: gensimple
 .PHONY: gen
 
-gensimple: api-gen go-generate cfgdoc-gen docsgen docsgen-cli
+marketgen:
+	swag init -dir market/mk20/http -g http.go  -o market/mk20/http --parseDependencyLevel 3 --parseDependency
+.PHONY: marketgen
+
+gensimple: api-gen go-generate cfgdoc-gen docsgen marketgen docsgen-cli
 	$(GOCC) run ./scripts/fiximports
 	go mod tidy
 .PHONY: gen

alertmanager/alerts.go

@@ -3,7 +3,6 @@ package alertmanager
 import (
 	"bytes"
 	"context"
-	"database/sql"
 	"fmt"
 	"math"
 	"strings"
@@ -13,6 +12,7 @@ import (
 	"github.com/dustin/go-humanize"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/samber/lo"
+	"github.com/yugabyte/pgx/v5"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -346,7 +346,7 @@ func (al *alerts) getAddresses() ([]address.Address, []address.Address, error) {
 		cfg := config.DefaultCurioConfig()
 		err := al.db.QueryRow(al.ctx, `SELECT config FROM harmony_config WHERE title=$1`, layer).Scan(&text)
 		if err != nil {
-			if strings.Contains(err.Error(), sql.ErrNoRows.Error()) {
+			if strings.Contains(err.Error(), pgx.ErrNoRows.Error()) {
 				return nil, nil, xerrors.Errorf("missing layer '%s' ", layer)
 			}
 			return nil, nil, xerrors.Errorf("could not read layer '%s': %w", layer, err)
@@ -731,7 +731,7 @@ func missingSectorCheck(al *alerts) {
 		SectorID int64 `db:"sector_num"`
 	}
 
-	err := al.db.Select(al.ctx, &sectors, `SELECT miner_id, sector_num  FROM sector_location WHERE sector_filetype = 2 GROUP BY miner_id, sector_num ORDER BY miner_id, sector_num`)
+	err := al.db.Select(al.ctx, &sectors, `SELECT miner_id, sector_num  FROM sector_location WHERE sector_filetype = ANY(ARRAY[2,8]) GROUP BY miner_id, sector_num ORDER BY miner_id, sector_num`)
 	if err != nil {
 		al.alertMap[Name].err = xerrors.Errorf("getting sealed sectors from database: %w", err)
 		return

alertmanager/plugin/slack_webhook.go

@@ -65,29 +65,60 @@ func (s *SlackWebhook) SendAlert(data *AlertPayload) error {
 
 	// Iterate through the map to construct the remaining blocks
 	for key, value := range data.Details {
-		// Split value into sentences by period followed by space
+		// Split value into sentences by period followed by space.
 		sentences := strings.Split(value.(string), ". ")
-		formattedValue := fmt.Sprintf("• *%s*\n", key)
 
-		// Add a bullet point before each trimmed sentence
+		// Add the key as the header for each block.
+		baseFormattedValue := fmt.Sprintf("• *%s*\n", key)
+		currentFormattedValue := baseFormattedValue
+
+		// Keep track of the character limit (3000) when adding sentences.
 		for _, sentence := range sentences {
-			trimmedSentence := strings.TrimSpace(sentence) // Trim leading and trailing spaces
+			trimmedSentence := strings.TrimSpace(sentence) // Trim leading and trailing spaces.
 			if trimmedSentence != "" {
-				formattedValue += fmt.Sprintf("• %s.\n", trimmedSentence) // Add period back and newline
+				// Add a bullet point and sentence, restoring the period and newline.
+				newSection := fmt.Sprintf("• %s.\n", trimmedSentence)
+
+				// Check if adding this section exceeds the 3000-character limit.
+				if len(currentFormattedValue)+len(newSection) > 3000 {
+					// If limit exceeds, add the currentFormattedValue block to payload and start a new block.
+					payload.Blocks = append(payload.Blocks,
+						Block{
+							Type: "section",
+							Text: &TextBlock{
+								Type: "mrkdwn",
+								Text: currentFormattedValue,
+							},
+						},
+						Block{
+							Type: "divider",
+						},
+					)
+
+					// Start a new formatted value with the baseFormattedValue.
+					currentFormattedValue = baseFormattedValue
+				}
+
+				// Append the newSection to the currentFormattedValue.
+				currentFormattedValue += newSection
 			}
 		}
-		payload.Blocks = append(payload.Blocks,
-			Block{
-				Type: "section",
-				Text: &TextBlock{
-					Type: "mrkdwn",
-					Text: formattedValue,
+
+		// Add the last block if it contains any content.
+		if currentFormattedValue != baseFormattedValue {
+			payload.Blocks = append(payload.Blocks,
+				Block{
+					Type: "section",
+					Text: &TextBlock{
+						Type: "mrkdwn",
+						Text: currentFormattedValue,
+					},
 				},
-			},
-			Block{
-				Type: "divider",
-			},
-		)
+				Block{
+					Type: "divider",
+				},
+			)
+		}
 	}
 
 	// Marshal the payload to JSON
@@ -163,7 +194,8 @@ func (s *SlackWebhook) SendAlert(data *AlertPayload) error {
 			}
 		})
 	if err != nil {
-		return fmt.Errorf("after %d retries,last error: %w", iter, err)
+		log.Errorw("Slack Webhook payload:", string(jsonData))
+		return fmt.Errorf("after %d retries,last error: %w, %s", iter, err, string(jsonData))
 	}
 	return nil
 }

cmd/curio/debug-ipni.go

@@ -48,7 +48,7 @@ var testDebugIpniChunks = &cli.Command{
 			return xerrors.Errorf("reading block: %w", err)
 		}
 
-		_, err = ck.Finish(c.Context, nil, cid.Undef)
+		_, err = ck.Finish(c.Context, nil, cid.Undef, false)
 		if err != nil {
 			return xerrors.Errorf("chunking CAR multihash iterator: %w", err)
 		}