Skip to content

Update IPFS endpoints #2055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/moody-cars-rule.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@graphprotocol/graph-cli': patch
---

updated IPFS endpoints, added deprecation notice when using old ones
2 changes: 1 addition & 1 deletion examples/aggregations/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"codegen": "graph codegen",
"create": "graph create example --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create example --node http://127.0.0.1:8020",
"deploy": "graph deploy example --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy example --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This (and all other package.json's below) should use --ipfs https://ipfs.thegraph.com

"deploy-local": "graph deploy example --ipfs http://127.0.0.1:5001 --node http://127.0.0.1:8020",
"test": "graph test"
},
Expand Down
2 changes: 1 addition & 1 deletion examples/arweave-blocks-transactions/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"codegen": "graph codegen",
"create": "graph create arweave-example --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create arweave-example --node http://localhost:8020",
"deploy": "graph deploy arweave-example --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy arweave-example --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy arweave-example -l v0.1.0 --ipfs http://localhost:5001 --node http://localhost:8020",
"remove-local": "graph remove arweave-example --node http://localhost:8020"
},
Expand Down
2 changes: 1 addition & 1 deletion examples/cosmos-block-filtering/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create cosmos-block-filtering --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create cosmos-block-filtering --node http://127.0.0.1:8020",
"deploy": "graph deploy cosmos-block-filtering --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy cosmos-block-filtering --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy cosmos-block-filtering -l v0.1.0 --ipfs http://127.0.0.1:5001 --node http://127.0.0.1:8020",
"prepare:cosmoshub": "mustache config/cosmoshub.json subgraph.template.yaml > subgraph.yaml",
"prepare:osmosis": "mustache config/osmosis.json subgraph.template.yaml > subgraph.yaml",
Expand Down
2 changes: 1 addition & 1 deletion examples/cosmos-osmosis-token-swaps/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create osmosis-token-swaps --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create osmosis-token-swaps --node http://0.0.0.0:8020",
"deploy": "graph deploy osmosis-token-swaps --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy osmosis-token-swaps --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy osmosis-token-swaps -l v0.1.0 --ipfs http://0.0.0.0:5001 --node http://0.0.0.0:8020",
"remove-local": "graph remove osmosis-token-swaps --node http://0.0.0.0:8020"
},
Expand Down
2 changes: 1 addition & 1 deletion examples/cosmos-validator-delegations/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create cosmos-validator-delegations --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create cosmos-validator-delegations --node http://127.0.0.1:8020",
"deploy": "graph deploy cosmos-validator-delegations --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy cosmos-validator-delegations --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy cosmos-validator-delegations -l v0.1.0 --ipfs http://127.0.0.1:5001 --node http://127.0.0.1:8020",
"prepare:cosmoshub": "mustache config/cosmoshub.json subgraph.template.yaml > subgraph.yaml",
"prepare:osmosis": "mustache config/osmosis.json subgraph.template.yaml > subgraph.yaml",
Expand Down
2 changes: 1 addition & 1 deletion examples/cosmos-validator-rewards/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create cosmos-validator-rewards --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create cosmos-validator-rewards --node http://127.0.0.1:8020",
"deploy": "graph deploy cosmos-validator-rewards --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy cosmos-validator-rewards --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy cosmos-validator-rewards -l v0.1.0 --ipfs http://127.0.0.1:5001 --node http://127.0.0.1:8020",
"prepare:cosmoshub": "mustache config/cosmoshub.json subgraph.template.yaml > subgraph.yaml",
"prepare:osmosis": "mustache config/osmosis.json subgraph.template.yaml > subgraph.yaml",
Expand Down
2 changes: 1 addition & 1 deletion examples/ethereum-gravatar/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"codegen": "graph codegen",
"create": "graph create example --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create example --node http://127.0.0.1:8020",
"deploy": "graph deploy example --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy example --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy example --ipfs http://127.0.0.1:5001 --node http://127.0.0.1:8020"
},
"devDependencies": {
Expand Down
2 changes: 1 addition & 1 deletion examples/near-blocks/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create example --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create example --node http://127.0.0.1:8020",
"deploy": "graph deploy example --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy example --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy example --ipfs http://localhost:5001 --node http://127.0.0.1:8020"
},
"devDependencies": {
Expand Down
2 changes: 1 addition & 1 deletion examples/near-receipts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"codegen": "graph codegen",
"create": "graph create example --node https://api.studio.thegraph.com/deploy/",
"create-local": "graph create example --node http://127.0.0.1:8020",
"deploy": "graph deploy example --ipfs https://api.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy": "graph deploy example --ipfs https://ipfs.thegraph.com/ipfs/ --node https://api.studio.thegraph.com/deploy/",
"deploy-local": "graph deploy example --ipfs http://localhost:5001 --node http://127.0.0.1:8020"
},
"devDependencies": {
Expand Down
80 changes: 54 additions & 26 deletions packages/cli/src/command-helpers/compiler.test.ts
Original file line number Diff line number Diff line change
@@ -1,42 +1,70 @@
import { describe, expect, it } from 'vitest';
import { appendApiVersionForGraph } from './compiler.js';
import { getGraphIpfsUrl } from './ipfs.js';

describe('appendApiVersionForGraph', { concurrent: true }, () => {
it('append /api/v0 to Prod URL with trailing slash', () => {
expect(appendApiVersionForGraph('https://api.thegraph.com/ipfs/')).toBe(
'https://api.thegraph.com/ipfs/api/v0',
);
const DEFAULT_IPFS_URL = 'https://ipfs.thegraph.com/ipfs/api/v0';

describe('getGraphIpfsUrl', { concurrent: true }, () => {
it('returns default URL when input is undefined', () => {
expect(getGraphIpfsUrl(undefined)).toEqual({
ipfsUrl: DEFAULT_IPFS_URL,
});
});

it('returns default URL when input is empty string', () => {
expect(getGraphIpfsUrl('')).toEqual({
ipfsUrl: DEFAULT_IPFS_URL,
});
});

it('returns input URL when valid and not deprecated', () => {
const validUrl = 'https://ipfs.network.example.com';
expect(getGraphIpfsUrl(validUrl)).toEqual({
ipfsUrl: validUrl,
});
});

it('trim trailing slash from valid url', () => {
const validUrl = 'https://ipfs.network.example.com/ipfs/';
expect(getGraphIpfsUrl(validUrl)).toEqual({
ipfsUrl: 'https://ipfs.network.example.com/ipfs',
});
});

it('append /api/v0 to Prod URL without trailing slash', () => {
expect(appendApiVersionForGraph('https://api.thegraph.com/ipfs')).toBe(
'https://api.thegraph.com/ipfs/api/v0',
);
it('returns default URL with warning for deprecated api.thegraph.com URL', () => {
const result = getGraphIpfsUrl('https://api.thegraph.com/ipfs/api/v0');
expect(result.ipfsUrl).toEqual(DEFAULT_IPFS_URL);
expect(result.warning).toContain('deprecated');
});

it('append /api/v0 to Staging URL without trailing slash', () => {
expect(appendApiVersionForGraph('https://staging.api.thegraph.com/ipfs')).toBe(
'https://staging.api.thegraph.com/ipfs/api/v0',
);
it('returns default URL with warning for deprecated ipfs.testnet.thegraph.com URL', () => {
const result = getGraphIpfsUrl('https://ipfs.testnet.thegraph.com/abc');
expect(result.ipfsUrl).toEqual(DEFAULT_IPFS_URL);
expect(result.warning).toContain('deprecated');
});

it('do nothing if Prod URL has /api/v0', () => {
expect(appendApiVersionForGraph('https://api.thegraph.com/ipfs/api/v0')).toBe(
'https://api.thegraph.com/ipfs/api/v0',
);
it('returns default URL with warning for deprecated ipfs.network.thegraph.com URL', () => {
const result = getGraphIpfsUrl('https://ipfs.network.thegraph.com/xyz');
expect(result.ipfsUrl).toEqual(DEFAULT_IPFS_URL);
expect(result.warning).toContain('deprecated');
});

it('do nothing if Prod URL has no /ipfs', () => {
expect(appendApiVersionForGraph('https://api.thegraph.com')).toBe('https://api.thegraph.com');
it('returns default URL with warning for invalid URL', () => {
const result = getGraphIpfsUrl('not-a-valid-url');
expect(result.ipfsUrl).toEqual(DEFAULT_IPFS_URL);
expect(result.warning).toContain('Invalid IPFS URL');
});

it('do nothing for non-graph endpoint', () => {
expect(appendApiVersionForGraph('https://ipfs.saihaj.dev/')).toBe('https://ipfs.saihaj.dev/');
it('preserves non-deprecated graph endpoints', () => {
const url = 'https://ipfs.thegraph.com/ipfs';
expect(getGraphIpfsUrl(url)).toEqual({
ipfsUrl: DEFAULT_IPFS_URL,
});
});

it('do nothing for non-graph endpoint ending with /ipfs', () => {
expect(appendApiVersionForGraph('https://ipfs.saihaj.dev/ipfs/')).toBe(
'https://ipfs.saihaj.dev/ipfs/',
);
it('preserves third-party IPFS endpoints', () => {
const url = 'https://ipfs.example.com/api';
expect(getGraphIpfsUrl(url)).toEqual({
ipfsUrl: url,
});
});
});
19 changes: 2 additions & 17 deletions packages/cli/src/command-helpers/compiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import Compiler from '../compiler/index.js';
import { GRAPH_CLI_SHARED_HEADERS } from '../constants.js';
import Protocol from '../protocols/index.js';
import { createIpfsClient } from '../utils.js';
import { getGraphIpfsUrl } from './ipfs.js';

interface CreateCompilerOptions {
ipfs: string | URL | undefined;
Expand All @@ -16,22 +17,6 @@ interface CreateCompilerOptions {
protocol: Protocol;
}

/**
* Appends /api/v0 to the end of a The Graph IPFS URL
*/
export function appendApiVersionForGraph(inputString: string) {
// Check if the input string is a valid The Graph IPFS URL
const pattern = /^(https?:\/\/)?([\w-]+\.)+thegraph\.com\/ipfs\/?$/;
if (pattern.test(inputString)) {
// account for trailing slash
if (inputString.endsWith('/')) {
return inputString.slice(0, -1) + '/api/v0';
}
return inputString + '/api/v0';
}
return inputString;
}

// Helper function to construct a subgraph compiler
export function createCompiler(
manifest: string,
Expand All @@ -57,7 +42,7 @@ The IPFS URL must be of the following format: http(s)://host[:port]/[path]`);
// Connect to the IPFS node (if a node address was provided)
const ipfsClient = ipfs
? createIpfsClient({
url: appendApiVersionForGraph(ipfs.toString()),
url: getGraphIpfsUrl(ipfs.toString()).ipfsUrl,
headers: {
...headers,
...GRAPH_CLI_SHARED_HEADERS,
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/command-helpers/file-resolver.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os from 'node:os';
import path from 'node:path';
import fs from 'fs-extra';
import { DEFAULT_IPFS_URL } from './ipfs.js';
import { getGraphIpfsUrl } from './ipfs.js';

export interface FileSource {
path: string;
Expand Down Expand Up @@ -39,7 +39,7 @@ export async function resolveFile(
try {
// If it's an IPFS hash (Qm...)
if (source.startsWith('Qm')) {
const response = await fetch(`${DEFAULT_IPFS_URL}/cat?arg=${source}`);
const response = await fetch(`${getGraphIpfsUrl().ipfsUrl}/cat?arg=${source}`);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is wrong. The /api/v0/cat endpoint is a POST endpoint and only works with GET atm. due to backwards compatibility. All GET requests should go to /ipfs/{cid}.

if (!response.ok) {
throw new Error(`failed to fetch from IPFS: ${response.statusText}`);
}
Expand Down
48 changes: 46 additions & 2 deletions packages/cli/src/command-helpers/ipfs.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,47 @@
const DEFAULT_IPFS_URL = 'https://api.thegraph.com/ipfs/api/v0' as const;
const DEFAULT_IPFS_URL = 'https://ipfs.thegraph.com/ipfs/api/v0' as const;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The IPFS URL should just be https://ipfs.thegraph.com

For GET requests (read a file), you'd append /ipfs/{cid}.

For POST requests (adding a file) you'd append /api/v0/add


export { DEFAULT_IPFS_URL };
/**
* Validates supplied IPFS URL and provides warnings for deprecated/invalid URLs
* @param ipfsUrl - The IPFS URL to validate, can be undefined
* @returns An object with the validated URL and optional warning message
*/
export function getGraphIpfsUrl(ipfsUrl?: string): { ipfsUrl: string; warning?: string } {
if (!ipfsUrl) {
return { ipfsUrl: DEFAULT_IPFS_URL };
}

// trim trailing slash
ipfsUrl = ipfsUrl.replace(/\/+$/, '');

try {
new URL(ipfsUrl);

const deprecatedPatterns = [
/^https?:\/\/ipfs\.testnet\.thegraph\.com.*/,
/^https?:\/\/ipfs\.network\.thegraph\.com.*/,
/^https?:\/\/api\.thegraph\.com\/ipfs.*/,
];

const isDeprecated = deprecatedPatterns.some(pattern => pattern.test(ipfsUrl));
if (isDeprecated) {
return {
ipfsUrl: DEFAULT_IPFS_URL,
warning: `IPFS URL ${ipfsUrl} is deprecated. Using default URL instead: ${DEFAULT_IPFS_URL}`,
};
}

// if default URL - make sure it ends with /api/v0
if (DEFAULT_IPFS_URL.startsWith(ipfsUrl)) {
return {
ipfsUrl: DEFAULT_IPFS_URL,
};
}

return { ipfsUrl };
} catch (e) {
return {
ipfsUrl: DEFAULT_IPFS_URL,
warning: `Invalid IPFS URL: ${ipfsUrl}. Using default URL instead: ${DEFAULT_IPFS_URL}`,
};
}
}
7 changes: 6 additions & 1 deletion packages/cli/src/commands/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { filesystem } from 'gluegun';
import { Args, Command, Flags } from '@oclif/core';
import { createCompiler } from '../command-helpers/compiler.js';
import * as DataSourcesExtractor from '../command-helpers/data-sources.js';
import { getGraphIpfsUrl } from '../command-helpers/ipfs.js';
import { updateSubgraphNetwork } from '../command-helpers/network.js';
import debug from '../debug.js';
import Protocol from '../protocols/index.js';
Expand Down Expand Up @@ -86,9 +87,13 @@ export default class BuildCommand extends Command {
const identifierName = protocol.getContract()!.identifierName();
await updateSubgraphNetwork(manifest, network, networkFile, identifierName);
}
const { ipfsUrl, warning } = getGraphIpfsUrl(ipfs);
if (warning) {
this.warn(warning);
}

const compiler = createCompiler(manifest, {
ipfs,
ipfs: ipfsUrl,
outputDir,
outputFormat,
skipMigrations,
Expand Down
11 changes: 8 additions & 3 deletions packages/cli/src/commands/codegen.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import path from 'node:path';
import { Args, Command, Flags } from '@oclif/core';
import * as DataSourcesExtractor from '../command-helpers/data-sources.js';
import { DEFAULT_IPFS_URL } from '../command-helpers/ipfs.js';
import { getGraphIpfsUrl } from '../command-helpers/ipfs.js';
import { assertGraphTsVersion, assertManifestApiVersion } from '../command-helpers/version.js';
import debug from '../debug.js';
import Protocol from '../protocols/index.js';
Expand Down Expand Up @@ -42,7 +42,7 @@ export default class CodegenCommand extends Command {
ipfs: Flags.string({
summary: 'IPFS node to use for fetching subgraph data.',
char: 'i',
default: DEFAULT_IPFS_URL,
default: getGraphIpfsUrl().ipfsUrl,
}),
'uncrashable-config': Flags.file({
summary: 'Directory for uncrashable config.',
Expand Down Expand Up @@ -89,6 +89,11 @@ export default class CodegenCommand extends Command {
this.error(e, { exit: 1 });
}

const { ipfsUrl, warning } = getGraphIpfsUrl(ipfs);
if (warning) {
this.warn(warning);
}

const generator = new TypeGenerator({
subgraphManifest: manifest,
outputDir,
Expand All @@ -97,7 +102,7 @@ export default class CodegenCommand extends Command {
uncrashable,
subgraphSources,
uncrashableConfig: uncrashableConfig || 'uncrashable-config.yaml',
ipfsUrl: ipfs,
ipfsUrl,
});

// Watch working directory for file updates or additions, trigger
Expand Down
Loading