Skip to content

Commit 3a9f22e

Browse files
LexLuthrLexLuthr
committed
basic mk12 scafolding
1 parent dbec7a0 commit 3a9f22e

File tree

17 files changed

+1247
-15
lines changed

17 files changed

+1247
-15
lines changed

‎cmd/curio/guidedsetup/shared_cbor_gen.go‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deps/config/doc_gen.go‎

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ require (
8585
go.opencensus.io v0.24.0
8686
go.uber.org/multierr v1.11.0
8787
go.uber.org/zap v1.27.0
88-
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
8988
golang.org/x/net v0.26.0
9089
golang.org/x/sync v0.7.0
9190
golang.org/x/sys v0.23.0
@@ -320,6 +319,7 @@ require (
320319
go.uber.org/mock v0.4.0 // indirect
321320
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
322321
golang.org/x/crypto v0.25.0 // indirect
322+
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
323323
golang.org/x/mod v0.17.0 // indirect
324324
golang.org/x/term v0.22.0 // indirect
325325
golang.org/x/time v0.5.0 // indirect

‎harmony/harmonydb/sql/20240730-market-migration.sql‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
CREATE TABLE market_mk12_deals (
2+
uuid TEXT NOT NULL,
3+
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
4+
signed_proposal_cid TEXT NOT NULL,
5+
proposal_signature BYTEA NOT NULL,
6+
proposal jsonb NOT NULL,
7+
piece_cid TEXT NOT NULL,
8+
piece_size BIGINT NOT NULL,
9+
offline BOOLEAN NOT NULL,
10+
verified BOOLEAN NOT NULL,
11+
sp_id BIGINT NOT NULL,
12+
start_epoch BIGINT NOT NULL,
13+
end_epoch BIGINT NOT NULL,
14+
client_peer_id TEXT NOT NULL,
15+
chain_deal_id BIGINT DEFAULT NULL,
16+
publish_cid TEXT DEFAULT NULL,
17+
length BIGINT DEFAULT NULL,
18+
fast_retrieval BOOLEAN NOT NULL,
19+
announce_to_ipni BOOLEAN NOT NULL,
20+
url TEXT DEFAULT NULL,
21+
url_headers jsonb NOT NULL DEFAULT '{}',
22+
error TEXT DEFAULT NULL,
23+
24+
primary key (uuid, sp_id, piece_cid, signed_proposal_cid),
25+
unique (uuid),
26+
unique (signed_proposal_cid)
27+
);
28+
29+
CREATE TABLE market_legacy_deals (
30+
signed_proposal_cid TEXT,
31+
proposal_signature BYTEA,
32+
proposal jsonb,
33+
piece_cid TEXT,
34+
piece_size BIGINT,
35+
offline BOOLEAN,
36+
verified BOOLEAN,
37+
sp_id BIGINT,
38+
start_epoch BIGINT,
39+
end_epoch BIGINT,
40+
publish_cid TEXT,
41+
fast_retrieval BOOLEAN,
42+
chain_deal_id BIGINT,
43+
created_at TIMESTAMPTZ,
44+
sector_num BIGINT,
45+
46+
primary key (sp_id, piece_cid, signed_proposal_cid)
47+
);
48+
49+
CREATE TABLE market_piece_metadata (
50+
piece_cid TEXT NOT NULL PRIMARY KEY,
51+
version INT NOT NULL DEFAULT 2,
52+
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
53+
indexed BOOLEAN NOT NULL DEFAULT FALSE,
54+
indexed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
55+
56+
constraint market_piece_meta_identity_key
57+
unique (piece_cid)
58+
);
59+
60+
CREATE TABLE market_piece_deal (
61+
id TEXT NOT NULL,
62+
piece_cid TEXT NOT NULL,
63+
boost_deal BOOLEAN NOT NULL,
64+
legacy_deal BOOLEAN NOT NULL DEFAULT FALSE,
65+
chain_deal_id BIGINT NOT NULL DEFAULT 0,
66+
sp_id BIGINT NOT NULL,
67+
sector_num BIGINT NOT NULL,
68+
piece_offset BIGINT NOT NULL,
69+
piece_length BIGINT NOT NULL,
70+
raw_size BIGINT NOT NULL,
71+
72+
primary key (sp_id, piece_cid, deal),
73+
constraint market_piece_deal_identity_key
74+
unique (sp_id, id)
75+
);
76+
77+
CREATE OR REPLACE FUNCTION process_piece_deal(
78+
_id TEXT,
79+
_piece_cid TEXT,
80+
_boost_deal BOOLEAN,
81+
_legacy_deal BOOLEAN DEFAULT FALSE,
82+
_chain_deal_id BIGINT DEFAULT 0,
83+
_sp_id BIGINT,
84+
_sector_num BIGINT,
85+
_piece_offset BIGINT,
86+
_piece_length BIGINT,
87+
_raw_size BIGINT
88+
)
89+
RETURNS VOID AS $$
90+
BEGIN
91+
-- Update or insert into market_piece_metadata
92+
INSERT INTO market_piece_metadata (piece_cid, indexed, indexed_at)
93+
VALUES (_piece_cid, TRUE, CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
94+
ON CONFLICT (piece_cid) DO UPDATE
95+
SET indexed = TRUE,
96+
indexed_at = CURRENT_TIMESTAMP AT TIME ZONE 'UTC';
97+
98+
-- Insert into market_piece_deal
99+
INSERT INTO market_piece_deal (
100+
id, piece_cid, boost_deal, legacy_deal, chain_deal_id,
101+
sp_id, sector_num, piece_offset, piece_length, raw_size
102+
)
103+
VALUES (
104+
_id, _piece_cid, _boost_deal, _legacy_deal, _chain_deal_id,
105+
_sp_id, _sector_num, _piece_offset, _piece_length, _raw_size
106+
)
107+
ON CONFLICT (sp_id, piece_cid, id) DO NOTHING;
108+
END;
109+
$$ LANGUAGE plpgsql;
110+
111+
CREATE TABLE market_mk12_storage_ask (
112+
price BIGINT NOT NULL,
113+
verified_price BIGINT NOT NULL,
114+
min_size BIGINT NOT NULL,
115+
max_size BIGINT NOT NULL,
116+
sp_id BIGINT NOT NULL,
117+
created_at BIGINT NOT NULL,
118+
expiry BIGINT NOT NULL,
119+
sequence BIGINT NOT NULL,
120+
unique (sp_id)
121+
);
122+
123+
CREATE TABLE market_mk12_deal_pipeline (
124+
uuid TEXT NOT NULL,
125+
sp_id BIGINT NOT NULL,
126+
started BOOLEAN DEFAULT FALSE,
127+
piece_cid TEXT NOT NULL,
128+
piece_size BOOLEAN NOT NULL,
129+
offline BOOLEAN NOT NULL,
130+
downloaded BOOLEAN DEFAULT FALSE,
131+
url TEXT DEFAULT NULL,
132+
headers jsonb NOT NULL DEFAULT '{}',
133+
file_size BIGINT DEFAULT NULL,
134+
commp_task_id BIGINT DEFAULT NULL,
135+
after_commp BOOLEAN DEFAULT FALSE,
136+
find_task_id BIGINT DEFAULT NULL,
137+
after_find BOOLEAN DEFAULT FALSE,
138+
psd_task_id BIGINT DEFAULT NULL,
139+
after_psd BOOLEAN DEFAULT FALSE,
140+
psd_wait_time TIMESTAMPTZ,
141+
find_deal_task_id BIGINT DEFAULT NULL,
142+
after_find_deal BOOLEAN DEFAULT FALSE,
143+
sector BIGINT,
144+
sector_offset BIGINT,
145+
sealed BOOLEAN DEFAULT FALSE,
146+
indexed BOOLEAN DEFAULT FALSE,
147+
148+
constraint market_mk12_deal_pipeline_identity_key unique (uuid)
149+
);
150+
151+
CREATE TABLE market_offline_urls (
152+
piece_cid TEXT NOT NULL,
153+
url TEXT NOT NULL,
154+
headers jsonb NOT NULL DEFAULT '{}',
155+
raw_size BIGINT NOT NULL,
156+
unique (piece_cid)
157+
);
158+
159+
CREATE TABLE market_indexing_tasks (
160+
id TEXT NOT NULL,
161+
sp_id BIGINT NOT NULL,
162+
sector_number BIGINT NOT NULL,
163+
reg_seal_proof INT NOT NULL,
164+
piece_offset BIGINT NOT NULL,
165+
piece_size BIGINT NOT NULL,
166+
raw_size BIGINT NOT NULL,
167+
piece_cid TEXT NOT NULL,
168+
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP AT TIME ZONE 'UTC',
169+
task_id BIGINT DEFAULT NULL,
170+
171+
constraint market_indexing_tasks_identity_key
172+
unique (id, sp_id, sector_number, piece_offset, piece_size, piece_cid, reg_seal_proof)
173+
)
174+
175+
CREATE TABLE libp2p_keys (
176+
sp_id BIGINT NOT NULL,
177+
priv_key BYTEA NOT NULL,
178+
listen_address TEXT NOT NULL,
179+
announce_address TEXT NOT NULL,
180+
no_announce_address TEXT NOT NULL,
181+
);
182+
183+
CREATE TABLE direct_deals (
184+
id TEXT,
185+
created_at TIMESTAMPTZ,
186+
piece_cid TEXT,
187+
piece_size BIGINT,
188+
cleanup_data BOOLEAN,
189+
client_address TEXT,
190+
provider_address TEXT,
191+
allocation_id BIGINT,
192+
start_epoch BIGINT,
193+
end_epoch BIGINT,
194+
inbound_file_path TEXT,
195+
inbound_file_size BIGINT,
196+
sector_id BIGINT,
197+
offset BIGINT,
198+
length BIGINT,
199+
announce_to_ipni BOOLEAN,
200+
keep_unsealed_copy BOOLEAN
201+
);
202+
203+
204+
205+

‎lib/indexing/indexstore/create.cql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE TABLE IF NOT EXISTS PayloadToPiece (
2+
PieceCid BLOB,
3+
PayloadMultihash BLOB,
4+
BlockOffset BIGINT,
5+
BlockSize BIGINT,
6+
PRIMARY KEY (PayloadMultihash, PieceCid)
7+
);
8+
9+
CREATE INDEX piece_to_payload ON PayloadToPiece(PieceCid, PayloadMultihash);

0 commit comments

Comments
 (0)