add url add command

Author: LexLuthr

cmd/curio/market.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"bufio"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
@@ -27,6 +31,7 @@ var marketCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		marketSealCmd,
 		marketImportdataCmd,
+		marketAddOfflineURLCmd,
 	},
 }
 
@@ -204,3 +209,162 @@ var marketImportdataCmd = &cli.Command{
 		return nil
 	},
 }
+
+var marketAddOfflineURLCmd = &cli.Command{
+	Name:  "add-url",
+	Usage: "Add URL to fetch data for offline deals",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "file",
+			Usage:    "CSV file location to use for multiple deal input. Each line in the file should be in the format 'uuid,raw size,url,header1,header2...'\"",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "header",
+			Aliases:  []string{"H"},
+			Usage:    "Custom `HEADER` to include in the HTTP request",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "url",
+			Aliases:  []string{"u"},
+			Usage:    "`URL` to send the request to",
+			Required: true,
+		},
+	},
+	ArgsUsage: "<deal UUID> <raw size/car size>",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.IsSet("file") && cctx.Args().Len() != 2 {
+			return xerrors.Errorf("incorrect number of arguments")
+		}
+
+		ctx := reqcontext.ReqContext(cctx)
+		dep, err := deps.GetDepsCLI(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		if cctx.IsSet("file") {
+			// Read file line by line
+			fileStr := cctx.String("file")
+			loc, err := homedir.Expand(fileStr)
+			if err != nil {
+				return err
+			}
+			file, err := os.Open(loc)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := scanner.Text()
+				// Extract pieceCid, pieceSize and MinerAddr from line
+				parts := strings.SplitN(line, ",", 4)
+				if parts[0] == "" || parts[1] == "" || parts[2] == "" {
+					return fmt.Errorf("empty column value in the input file at %s", line)
+				}
+
+				uuid := parts[0]
+				size, err := strconv.ParseInt(parts[1], 10, 64)
+				if err != nil {
+					return fmt.Errorf("failed to parse size %w", err)
+				}
+
+				url := parts[2]
+
+				if parts[3] != "" {
+					header := http.Header{}
+					for _, s := range strings.Split(parts[3], ",") {
+						key, value, found := strings.Cut(s, ":")
+						if !found {
+							return fmt.Errorf("invalid header format, expected key:value")
+						}
+						header.Set(strings.TrimSpace(key), strings.TrimSpace(value))
+					}
+
+					hdr, err := json.Marshal(header)
+					if err != nil {
+						return xerrors.Errorf("marshalling headers: %w", err)
+					}
+					_, err = dep.DB.Exec(ctx, `INSERT INTO market_offline_urls (
+								uuid,
+								url,
+								headers,
+								raw_size
+							) VALUES ($1, $2, $3, $4);`,
+						uuid, url, hdr, size)
+					if err != nil {
+						return xerrors.Errorf("adding details to DB: %w", err)
+					}
+				} else {
+					_, err = dep.DB.Exec(ctx, `INSERT INTO market_offline_urls (
+								uuid,
+								url,
+								raw_size
+							) VALUES ($1, $2, $3, $4);`,
+						uuid, url, size)
+					if err != nil {
+						return xerrors.Errorf("adding details to DB: %w", err)
+					}
+				}
+
+				if err := scanner.Err(); err != nil {
+					return err
+				}
+			}
+		}
+
+		url := cctx.String("url")
+
+		headerValue := cctx.StringSlice("header")
+
+		uuid := cctx.Args().First()
+
+		sizeStr := cctx.Args().Get(1)
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			return xerrors.Errorf("parsing size: %w", err)
+		}
+
+		if cctx.IsSet("header") {
+			// Split the header into key-value
+			header := http.Header{}
+			for _, s := range headerValue {
+				key, value, found := strings.Cut(s, ":")
+				if !found {
+					return fmt.Errorf("invalid header format, expected key:value")
+				}
+				header.Set(strings.TrimSpace(key), strings.TrimSpace(value))
+			}
+
+			hdr, err := json.Marshal(header)
+			if err != nil {
+				return xerrors.Errorf("marshalling headers: %w", err)
+			}
+
+			_, err = dep.DB.Exec(ctx, `INSERT INTO market_offline_urls (
+								uuid,
+								url,
+								headers,
+								raw_size
+							) VALUES ($1, $2, $3, $4);`,
+				uuid, url, hdr, size)
+			if err != nil {
+				return xerrors.Errorf("adding details to DB: %w", err)
+			}
+		} else {
+			_, err = dep.DB.Exec(ctx, `INSERT INTO market_offline_urls (
+								uuid,
+								url,
+								raw_size
+							) VALUES ($1, $2, $3, $4);`,
+				uuid, url, size)
+			if err != nil {
+				return xerrors.Errorf("adding details to DB: %w", err)
+			}
+		}
+
+		return nil
+	},
+}

documentation/en/curio-cli/curio.md

@@ -549,6 +549,7 @@ USAGE:
 COMMANDS:
    seal         start sealing a deal sector early
    import-data  Import data for offline deal
+   add-url      Add URL to fetch data for offline deals
    help, h      Shows a list of commands or help for one command
 
 OPTIONS:
@@ -582,6 +583,21 @@ OPTIONS:
    --help, -h     show help
 ```
 
+### curio market add-url
+```
+NAME:
+   curio market add-url - Add URL to fetch data for offline deals
+
+USAGE:
+   curio market add-url [command options] <deal UUID> <raw size/car size>
+
+OPTIONS:
+   --file value                CSV file location to use for multiple deal input. Each line in the file should be in the format 'uuid,raw size,url,header1,header2...'"
+   --header HEADER, -H HEADER  Custom HEADER to include in the HTTP request
+   --url URL, -u URL           URL to send the request to
+   --help, -h                  show help
+```
+
 ## curio fetch-params
 ```
 NAME:

harmony/harmonydb/sql/20240730-market-migration.sql renamed to harmony/harmonydb/sql/20240731-market-migration.sql

@@ -230,14 +230,17 @@ $$ LANGUAGE plpgsql;
 -- This table can be used to track remote piece for offline deals
 -- The entries must be created by users
 CREATE TABLE market_offline_urls (
-    piece_cid TEXT NOT NULL,
+    uuid TEXT NOT NULL,
 
     url TEXT NOT NULL,
     headers jsonb NOT NULL DEFAULT '{}',
 
     raw_size BIGINT NOT NULL,
 
-    unique (piece_cid)
+    CONSTRAINT market_offline_urls_uuid_fk FOREIGN KEY (uuid)
+        REFERENCES market_mk12_deal_pipeline (uuid)
+        ON DELETE CASCADE,
+    CONSTRAINT market_offline_urls_uuid_unique UNIQUE (uuid)
 );
 
 -- This table is used for coordinating libp2p nodes

tasks/storage-market/storage_market.go

@@ -378,7 +378,7 @@ func (d *CurioStorageDealMarket) findURLForOfflineDeals(ctx context.Context, dea
 	}
 
 	err := d.db.Select(ctx, &goUrls, `SELECT url, headers, raw_size   
-								FROM market_offline_urls WHERE piece_cid = $1`, pcid)
+							FROM market_offline_urls WHERE piece_cid = $1 AND uuid = $2`, pcid, deal)
 
 	if err != nil {
 		return false, xerrors.Errorf("getting url and headers from db: %w", err)