Skip to content

Commit f5a1e6e

Browse files
SgtPookiCopilotredpanda-f
committed
feat: allow passing metadata via cli commands (#226)
* feat: allow passing metadata via cli commands * chore: cleanup code, add jsdocs * Update src/utils/cli-options-metadata.ts Co-authored-by: Copilot <[email protected]> * Update src/utils/cli-options-metadata.ts Co-authored-by: Copilot <[email protected]> * Update src/utils/cli-options-metadata.ts Co-authored-by: Copilot <[email protected]> * Update src/utils/cli-options-metadata.ts Co-authored-by: Copilot <[email protected]> * chore: fix lint * fix: session keys without CreateDataSet permission can add pieces (#234) * fix: setupSessionKey * fix: creation issues * update: resolve comments * fix: createDataSet permissions when using session --------- Co-authored-by: Russell Dempsey <[email protected]> * refactor: metadata->pieceMetadata --------- Co-authored-by: Copilot <[email protected]> Co-authored-by: RedPanda <[email protected]>
1 parent 40c400b commit f5a1e6e

File tree

22 files changed

+744
-25
lines changed

22 files changed

+744
-25
lines changed

‎src/add/add.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import pino from 'pino'
1111
import { warnAboutCDNPricingLimitations } from '../common/cdn-warning.js'
1212
import { TELEMETRY_CLI_APP_NAME } from '../common/constants.js'
1313
import { displayUploadResults, performAutoFunding, performUpload, validatePaymentSetup } from '../common/upload-flow.js'
14+
import { normalizeMetadataConfig } from '../core/metadata/index.js'
1415
import {
1516
cleanupSynapseService,
1617
createStorageContext,
@@ -69,6 +70,11 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
6970

7071
const spinner = createSpinner()
7172

73+
const { pieceMetadata, dataSetMetadata } = normalizeMetadataConfig({
74+
pieceMetadata: options.pieceMetadata,
75+
dataSetMetadata: options.dataSetMetadata,
76+
})
77+
7278
// Initialize logger (silent for CLI output)
7379
const logger = pino({
7480
level: process.env.LOG_LEVEL || 'silent',
@@ -110,6 +116,9 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
110116

111117
// Parse authentication options from CLI and environment
112118
const config = parseCLIAuth(options)
119+
if (dataSetMetadata) {
120+
config.dataSetMetadata = dataSetMetadata
121+
}
113122
if (withCDN) config.withCDN = true
114123

115124
// Initialize just the Synapse SDK
@@ -163,8 +172,11 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
163172
// Parse provider selection from CLI options and environment variables
164173
const providerOptions = parseProviderOptions(options)
165174

166-
const { storage, providerInfo } = await createStorageContext(synapse, logger, {
175+
const storageContextOptions: Parameters<typeof createStorageContext>[2] = {
167176
...providerOptions,
177+
dataset: {
178+
...(dataSetMetadata && { metadata: dataSetMetadata }),
179+
},
168180
callbacks: {
169181
onProviderSelected: (provider) => {
170182
spinner.message(`Connecting to storage provider: ${provider.name || provider.serviceProvider}...`)
@@ -177,7 +189,9 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
177189
}
178190
},
179191
},
180-
})
192+
}
193+
194+
const { storage, providerInfo } = await createStorageContext(synapse, logger, storageContextOptions)
181195

182196
spinner.stop(`${pc.green('✓')} Storage context ready`)
183197
log.spinnerSection('Storage Context', [
@@ -194,6 +208,7 @@ export async function runAdd(options: AddOptions): Promise<AddResult> {
194208
fileSize: carSize,
195209
logger,
196210
spinner,
211+
...(pieceMetadata && { metadata: pieceMetadata }),
197212
})
198213

199214
// Display results

‎src/add/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ export interface AddOptions extends CLIAuthOptions {
66
bare?: boolean
77
/** Auto-fund: automatically ensure minimum 30 days of runway */
88
autoFund?: boolean
9+
/** Piece metadata attached to each upload */
10+
pieceMetadata?: Record<string, string>
11+
/** Data set metadata applied when creating or updating the storage context */
12+
dataSetMetadata?: Record<string, string>
913
}
1014

1115
export interface AddResult {

‎src/commands/add.ts‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { runAdd } from '../add/add.js'
33
import type { AddOptions } from '../add/types.js'
44
import { MIN_RUNWAY_DAYS } from '../common/constants.js'
55
import { addAuthOptions, addProviderOptions } from '../utils/cli-options.js'
6+
import { addMetadataOptions, resolveMetadataOptions } from '../utils/cli-options-metadata.js'
67

78
export const addCommand = new Command('add')
89
.description('Add a file or directory to Filecoin via Synapse (creates UnixFS CAR)')
@@ -11,11 +12,21 @@ export const addCommand = new Command('add')
1112
.option('--auto-fund', `Automatically ensure minimum ${MIN_RUNWAY_DAYS} days of runway before upload`)
1213
.action(async (path: string, options) => {
1314
try {
15+
const {
16+
metadata: _metadata,
17+
dataSetMetadata: _dataSetMetadata,
18+
datasetMetadata: _datasetMetadata,
19+
'8004Type': _erc8004Type,
20+
'8004Agent': _erc8004Agent,
21+
...addOptionsFromCli
22+
} = options
23+
const { pieceMetadata, dataSetMetadata } = resolveMetadataOptions(options, { includeErc8004: true })
24+
1425
const addOptions: AddOptions = {
15-
...options,
26+
...addOptionsFromCli,
1627
filePath: path,
17-
bare: options.bare,
18-
autoFund: options.autoFund,
28+
...(pieceMetadata && { pieceMetadata }),
29+
...(dataSetMetadata && { dataSetMetadata }),
1930
}
2031

2132
await runAdd(addOptions)
@@ -27,3 +38,4 @@ export const addCommand = new Command('add')
2738

2839
addAuthOptions(addCommand)
2940
addProviderOptions(addCommand)
41+
addMetadataOptions(addCommand, { includePieceMetadata: true, includeDataSetMetadata: true, includeErc8004: true })

‎src/commands/data-set.ts‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { Command } from 'commander'
22
import { runDataSetDetailsCommand, runDataSetListCommand } from '../data-set/run.js'
33
import type { DataSetCommandOptions, DataSetListCommandOptions } from '../data-set/types.js'
44
import { addAuthOptions, addProviderOptions } from '../utils/cli-options.js'
5+
import { addMetadataOptions, resolveMetadataOptions } from '../utils/cli-options-metadata.js'
56

67
export const dataSetCommand = new Command('data-set')
78
.alias('dataset')
@@ -32,16 +33,27 @@ export const dataSetListCommand = new Command('list')
3233
.alias('ls')
3334
.description('List all data sets for the configured account')
3435
.option('--all', 'Show all data sets, not just the ones created with filecoin-pin', false)
35-
.action(async (options: DataSetListCommandOptions) => {
36+
.action(async (options) => {
3637
try {
37-
await runDataSetListCommand(options)
38+
const {
39+
dataSetMetadata: _dataSetMetadata,
40+
datasetMetadata: _datasetMetadata,
41+
...dataSetListOptionsFromCli
42+
} = options
43+
const { dataSetMetadata } = resolveMetadataOptions(options)
44+
const normalizedOptions: DataSetListCommandOptions = {
45+
...dataSetListOptionsFromCli,
46+
...(dataSetMetadata ? { dataSetMetadata } : {}),
47+
}
48+
await runDataSetListCommand(normalizedOptions)
3849
} catch (error) {
3950
console.error('Data set list command failed:', error instanceof Error ? error.message : error)
4051
process.exit(1)
4152
}
4253
})
4354
addAuthOptions(dataSetListCommand)
4455
addProviderOptions(dataSetListCommand)
56+
addMetadataOptions(dataSetListCommand, { includePieceMetadata: false, includeDataSetMetadata: true })
4557

4658
dataSetCommand.addCommand(dataSetShowCommand)
4759
dataSetCommand.addCommand(dataSetListCommand)

‎src/commands/import.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,29 @@ import { MIN_RUNWAY_DAYS } from '../common/constants.js'
33
import { runCarImport } from '../import/import.js'
44
import type { ImportOptions } from '../import/types.js'
55
import { addAuthOptions, addProviderOptions } from '../utils/cli-options.js'
6+
import { addMetadataOptions, resolveMetadataOptions } from '../utils/cli-options-metadata.js'
67

78
export const importCommand = new Command('import')
89
.description('Import an existing CAR file to Filecoin via Synapse')
910
.argument('<file>', 'Path to the CAR file to import')
1011
.option('--auto-fund', `Automatically ensure minimum ${MIN_RUNWAY_DAYS} days of runway before upload`)
1112
.action(async (file: string, options) => {
1213
try {
14+
const {
15+
metadata: _metadata,
16+
dataSetMetadata: _dataSetMetadata,
17+
datasetMetadata: _datasetMetadata,
18+
'8004Type': _erc8004Type,
19+
'8004Agent': _erc8004Agent,
20+
...importOptionsFromCli
21+
} = options
22+
23+
const { pieceMetadata, dataSetMetadata } = resolveMetadataOptions(options, { includeErc8004: true })
1324
const importOptions: ImportOptions = {
14-
...options,
25+
...importOptionsFromCli,
1526
filePath: file,
16-
autoFund: options.autoFund,
27+
...(pieceMetadata && { pieceMetadata }),
28+
...(dataSetMetadata && { dataSetMetadata }),
1729
}
1830

1931
await runCarImport(importOptions)
@@ -25,3 +37,4 @@ export const importCommand = new Command('import')
2537

2638
addAuthOptions(importCommand)
2739
addProviderOptions(importCommand)
40+
addMetadataOptions(importCommand, { includePieceMetadata: true, includeDataSetMetadata: true, includeErc8004: true })

‎src/common/upload-flow.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ export interface UploadFlowOptions {
4646
* Optional spinner for progress updates
4747
*/
4848
spinner?: Spinner
49+
50+
/**
51+
* Optional metadata attached to the upload request
52+
*/
53+
pieceMetadata?: Record<string, string>
4954
}
5055

5156
export interface UploadFlowResult extends SynapseUploadResult {
@@ -255,7 +260,7 @@ export async function performUpload(
255260
rootCid: CID,
256261
options: UploadFlowOptions
257262
): Promise<UploadFlowResult> {
258-
const { contextType, logger, spinner } = options
263+
const { contextType, logger, spinner, pieceMetadata } = options
259264

260265
// Create spinner flow manager for tracking all operations
261266
const flow = createSpinnerFlow(spinner)
@@ -273,6 +278,7 @@ export async function performUpload(
273278
const uploadResult = await executeUpload(synapseService, carData, rootCid, {
274279
logger,
275280
contextId: `${contextType}-${Date.now()}`,
281+
...(pieceMetadata && { pieceMetadata }),
276282
onProgress(event) {
277283
switch (event.type) {
278284
case 'onUploadComplete': {

‎src/core/metadata/index.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
export const ERC8004_TYPES = ['registration', 'validationrequest', 'validationresponse', 'feedback'] as const
2+
3+
export type ERC8004Type = (typeof ERC8004_TYPES)[number]
4+
5+
export interface MetadataConfigInput {
6+
pieceMetadata?: Record<string, string> | undefined
7+
dataSetMetadata?: Record<string, string> | undefined
8+
erc8004Type?: ERC8004Type
9+
erc8004Agent?: string
10+
}
11+
12+
export interface MetadataConfigResult {
13+
pieceMetadata?: Record<string, string> | undefined
14+
dataSetMetadata?: Record<string, string> | undefined
15+
}
16+
17+
export function normalizeMetadataConfig(input: MetadataConfigInput): MetadataConfigResult {
18+
const pieceMetadata = sanitizeRecord(input.pieceMetadata)
19+
const dataSetMetadata = sanitizeRecord(input.dataSetMetadata)
20+
21+
if ((input.erc8004Type && !input.erc8004Agent) || (!input.erc8004Type && input.erc8004Agent)) {
22+
throw new Error('Both erc8004Type and erc8004Agent must be provided together')
23+
}
24+
25+
if (input.erc8004Type && input.erc8004Agent) {
26+
const key = `8004${input.erc8004Type}`
27+
mergeRecord(pieceMetadata, key, input.erc8004Agent, 'ERC-8004 metadata', 'metadata')
28+
mergeRecord(dataSetMetadata, 'erc8004Files', '', 'ERC-8004 metadata', 'data set metadata')
29+
}
30+
31+
return {
32+
pieceMetadata: Object.keys(pieceMetadata).length > 0 ? pieceMetadata : undefined,
33+
dataSetMetadata: Object.keys(dataSetMetadata).length > 0 ? dataSetMetadata : undefined,
34+
}
35+
}
36+
37+
function sanitizeRecord(record: Record<string, string> | undefined): Record<string, string> {
38+
if (record == null) {
39+
return {}
40+
}
41+
42+
const sanitized: Record<string, string> = {}
43+
44+
for (const [rawKey, rawValue] of Object.entries(record)) {
45+
const key = rawKey.trim()
46+
if (key === '') {
47+
throw new Error('Metadata keys must be non-empty strings')
48+
}
49+
50+
if (typeof rawValue !== 'string') {
51+
throw new Error(`Metadata value for "${key}" must be a string`)
52+
}
53+
54+
sanitized[key] = rawValue
55+
}
56+
57+
return sanitized
58+
}
59+
60+
function mergeRecord(
61+
record: Record<string, string>,
62+
key: string,
63+
value: string,
64+
sourceLabel: string,
65+
conflictLabel: string
66+
): void {
67+
const existingValue = record[key]
68+
if (existingValue != null && existingValue !== value) {
69+
throw new Error(
70+
`Conflicting metadata for "${key}": ${sourceLabel} tried to set "${value}" but ${conflictLabel} already set "${existingValue}".`
71+
)
72+
}
73+
74+
record[key] = value
75+
}

‎src/core/synapse/index.ts‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ interface BaseSynapseConfig {
5050
/** Optional override for WarmStorage contract address */
5151
warmStorageAddress?: string | undefined
5252
withCDN?: boolean | undefined
53+
/** Default metadata to apply when creating or reusing datasets */
54+
dataSetMetadata?: Record<string, string>
5355
/**
5456
* Telemetry configuration.
5557
* Defaults to enabled unless explicitly disabled.
@@ -271,16 +273,43 @@ async function setupSessionKey(synapse: Synapse, sessionWallet: Wallet, logger:
271273
const now = Math.floor(Date.now() / 1000)
272274
const bufferTime = 30 * 60 // 30 minutes in seconds
273275
const minValidTime = now + bufferTime
274-
const createExpiry = Number(expiries[CREATE_DATA_SET_TYPEHASH])
275-
const addExpiry = Number(expiries[ADD_PIECES_TYPEHASH])
276+
const createDataSetExpiry = Number(expiries[CREATE_DATA_SET_TYPEHASH])
277+
const addPiecesExpiry = Number(expiries[ADD_PIECES_TYPEHASH])
276278

277-
if (createExpiry <= minValidTime || addExpiry <= minValidTime) {
279+
// For CREATE_DATA_SET:
280+
// - 0 means no permission granted (OK - can still add to existing datasets)
281+
// - > 0 but < minValidTime means expired/expiring (ERROR)
282+
// - >= minValidTime means valid (OK)
283+
const hasCreateDataSetPermission = createDataSetExpiry > 0
284+
const isCreateDataSetPermissionUnavailable = hasCreateDataSetPermission && createDataSetExpiry < minValidTime
285+
286+
// For ADD_PIECES:
287+
// - Must always have valid permission
288+
const isAddPiecesPermissionUnavailable = addPiecesExpiry <= minValidTime
289+
290+
if (isCreateDataSetPermissionUnavailable) {
278291
throw new Error(
279-
`Session key expired or expiring soon (requires 30+ minutes validity). CreateDataSet: ${new Date(createExpiry * 1000).toISOString()}, AddPieces: ${new Date(addExpiry * 1000).toISOString()}`
292+
`Session key expired or expiring soon (requires 30+ minutes validity). CreateDataSet: ${new Date(createDataSetExpiry * 1000).toISOString()}`
280293
)
281294
}
282295

283-
logger.info({ event: 'synapse.session_key.verified', createExpiry, addExpiry }, 'Session key verified')
296+
if (isAddPiecesPermissionUnavailable) {
297+
throw new Error(
298+
`Session key expired or expiring soon (requires 30+ minutes validity). AddPieces: ${new Date(addPiecesExpiry * 1000).toISOString()}`
299+
)
300+
}
301+
302+
if (!hasCreateDataSetPermission) {
303+
logger.info(
304+
{ event: 'synapse.session_key.limited_permissions' },
305+
'Session key can only add pieces to existing datasets (no CREATE_DATA_SET permission)'
306+
)
307+
}
308+
309+
logger.info(
310+
{ event: 'synapse.session_key.verified', createExpiry: createDataSetExpiry, addExpiry: addPiecesExpiry },
311+
'Session key verified'
312+
)
284313

285314
synapse.setSession(sessionKey)
286315
logger.info({ event: 'synapse.session_key.activated' }, 'Session key activated')
@@ -427,6 +456,22 @@ export async function createStorageContext(
427456
'Connecting to existing dataset'
428457
)
429458
} else if (options?.dataset?.createNew === true) {
459+
// If explicitly creating a new dataset in session key mode, verify we have permission
460+
if (isSessionKeyMode(synapse)) {
461+
const signer = synapse.getSigner()
462+
const sessionKey = synapse.createSessionKey(signer)
463+
464+
const expiries = await sessionKey.fetchExpiries([CREATE_DATA_SET_TYPEHASH])
465+
const createDataSetExpiry = Number(expiries[CREATE_DATA_SET_TYPEHASH])
466+
467+
if (createDataSetExpiry === 0) {
468+
throw new Error(
469+
'Cannot create new dataset: Session key does not have CREATE_DATA_SET permission. ' +
470+
'Either use an existing dataset or obtain a session key with dataset creation rights.'
471+
)
472+
}
473+
}
474+
430475
sdkOptions.forceCreateDataSet = true
431476
logger.info({ event: 'synapse.storage.dataset.create_new' }, 'Forcing creation of new dataset')
432477
}
@@ -568,7 +613,21 @@ export async function setupSynapse(
568613
const synapse = await initializeSynapse(config, logger)
569614

570615
// Create storage context
571-
const { storage, providerInfo } = await createStorageContext(synapse, logger, options)
616+
let storageOptions = options ? { ...options } : undefined
617+
if (config.dataSetMetadata && Object.keys(config.dataSetMetadata).length > 0) {
618+
storageOptions = {
619+
...(storageOptions ?? {}),
620+
dataset: {
621+
...(storageOptions?.dataset ?? {}),
622+
metadata: {
623+
...config.dataSetMetadata,
624+
...(storageOptions?.dataset?.metadata ?? {}),
625+
},
626+
},
627+
}
628+
}
629+
630+
const { storage, providerInfo } = await createStorageContext(synapse, logger, storageOptions)
572631

573632
return { synapse, storage, providerInfo }
574633
}

0 commit comments

Comments
 (0)