diff --git a/dappmanager-grafana-dashboard.json b/dappmanager-grafana-dashboard.json index 93d27c935..39d0532ec 100644 --- a/dappmanager-grafana-dashboard.json +++ b/dappmanager-grafana-dashboard.json @@ -195,38 +195,22 @@ "mode": "fixed" }, "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "light-red", - "index": 0, - "text": "OFF" - }, - "1": { - "color": "light-green", - "index": 1, - "text": "ON" - } - }, - "type": "value" - } - ], - "max": 1, + "mappings": [], + "max": 10, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "red" }, { - "color": "red", - "value": 80 + "color": "green", + "value": 1 } ] }, - "unit": "bool_on_off" + "unit": "short" }, "overrides": [] }, @@ -256,13 +240,13 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "dappmanager_ipfs_client_target_local{instance=\"dappmanager.dappnode:80\",ipfsClientTargetLocal=\"local\"}", + "expr": "dappmanager_ipfs_gateway_count{instance=\"dappmanager.dappnode:80\",gatewayCount=\"count\"}", "interval": "", - "legendFormat": "Ipfs Client Local (on / off)", + "legendFormat": "IPFS Gateways Configured", "refId": "A" } ], - "title": "Ipfs Client Local", + "title": "IPFS Gateways", "type": "gauge" }, { diff --git a/packages/admin-ui/src/__mock-backend__/index.ts b/packages/admin-ui/src/__mock-backend__/index.ts index facf1a69e..0979f9d98 100644 --- a/packages/admin-ui/src/__mock-backend__/index.ts +++ b/packages/admin-ui/src/__mock-backend__/index.ts @@ -1,4 +1,4 @@ -import { IpfsClientTarget, PortProtocol, Routes } from "@dappnode/types"; +import { PortProtocol, Routes } from "@dappnode/types"; import { autoUpdate } from "./autoUpdate"; import { devices } from "./devices"; import { fetchPkgsData } from "./fetchPkgsData"; @@ -221,10 +221,14 @@ export const otherCalls: Omit = { { lv_name: "swap_1", vg_name: "rootvg", lv_size: "976.00m" } ], lvmDiskSpaceExtend: async () => "Successfully extended LVM disk space", - ipfsClientTargetSet: async () => {}, - ipfsClientTargetGet: async () => ({ - ipfsClientTarget: IpfsClientTarget.remote, - ipfsGateway: "https://ipfs-gateway.dappnode.net" + ipfsGatewayUrlsSet: async () => {}, + ipfsGatewayUrlsGet: async () => ({ + ipfsGatewayUrls: [ + "https://ipfs-gateway.dappnode.net", + "https://ipfs-gateway-dev.dappnode.net", + "https://gateway.pinata.cloud", + "https://cloudflare-ipfs.com" + ] }), enableEthicalMetrics: async () => {}, getEthicalMetricsConfig: async () => ({ diff --git a/packages/admin-ui/src/components/IpfsClient.tsx b/packages/admin-ui/src/components/IpfsClient.tsx index 792fc4238..d69d00ab3 100644 --- a/packages/admin-ui/src/components/IpfsClient.tsx +++ b/packages/admin-ui/src/components/IpfsClient.tsx @@ -1,80 +1,38 @@ import React from "react"; -import "./multiClient.scss"; -import { IpfsClientTarget } from "@dappnode/types"; -import Card from "components/Card"; -import { joinCssClass } from "utils/css"; import Input from "./Input"; import { IPFS_DAPPNODE_GATEWAY, IPFS_GATEWAY_CHECKER } from "params"; import RenderMarkdown from "./RenderMarkdown"; -interface IpfsClientData { - title: string; - description: string; - option: IpfsClientTarget; -} +const description = `Configure the IPFS gateway URLs used for fetching Dappnode packages. The system will try your local IPFS node first, then race all configured gateways simultaneously - the first to respond wins. + +Enter multiple URLs separated by commas. You can find public gateways at [${IPFS_GATEWAY_CHECKER}](${IPFS_GATEWAY_CHECKER}). -const clients: IpfsClientData[] = [ - { - title: "Remote", - description: `Public IPFS node API mantained by Dappnode [${IPFS_DAPPNODE_GATEWAY}](${IPFS_DAPPNODE_GATEWAY}) or choose one from [${IPFS_GATEWAY_CHECKER}](${IPFS_GATEWAY_CHECKER})`, - option: IpfsClientTarget.remote - }, - { - title: "Local", - description: "Your own IPFS node w/out 3rd parties", - option: IpfsClientTarget.local - } -]; +Default gateway: [${IPFS_DAPPNODE_GATEWAY}](${IPFS_DAPPNODE_GATEWAY})`; /** - * View to chose or change the IPFS client - * There are two main options: - * - Remote - * - Local + * Simple component to configure IPFS gateway URLs */ export function IpfsClient({ - clientTarget: selectedClientTarget, - gatewayTarget, - onClientTargetChange, - onGatewayTargetChange + gatewayUrls, + onGatewayUrlsChange }: { - clientTarget: IpfsClientTarget | null; - gatewayTarget: string | null; - onClientTargetChange: (newTarget: IpfsClientTarget) => void; - onGatewayTargetChange: (newTarget: string) => void; + /** Comma-separated gateway URLs string for display/editing */ + gatewayUrls: string | null; + /** Callback receives comma-separated string of URLs */ + onGatewayUrlsChange: (newUrls: string) => void; }) { return ( -
- {clients - .filter(({ option }) => option.length > 0) - .map(({ title, description, option }) => { - const selected = selectedClientTarget && option === selectedClientTarget; - - return ( - { - // Prevent over-riding the options onClientTargetChange call - if (!selected) onClientTargetChange(option); - }} - > -
{title}
-
- -
+
+
+ +
- {option === "remote" && ( - - )} - - ); - })} + +
); } diff --git a/packages/admin-ui/src/pages/system/components/Ipfs/IpfsNode.tsx b/packages/admin-ui/src/pages/system/components/Ipfs/IpfsNode.tsx index 68ae74d12..bfef6c7fa 100644 --- a/packages/admin-ui/src/pages/system/components/Ipfs/IpfsNode.tsx +++ b/packages/admin-ui/src/pages/system/components/Ipfs/IpfsNode.tsx @@ -1,6 +1,5 @@ import React, { useState, useEffect } from "react"; import { useApi, api } from "api"; -import { IpfsClientTarget } from "@dappnode/types"; import { withToast } from "components/toast/Toast"; import { IpfsClient } from "components/IpfsClient"; import Button from "components/Button"; @@ -9,73 +8,98 @@ import { forumUrl } from "params"; import Card from "components/Card"; import SubTitle from "components/SubTitle"; +/** + * Converts array of URLs to comma-separated string for display + */ +function urlsToString(urls: string[]): string { + return urls.join(", "); +} + +/** + * Parses comma-separated string to array of URLs, trimming whitespace + */ +function stringToUrls(str: string): string[] { + return str + .split(",") + .map((url) => url.trim()) + .filter((url) => url.length > 0); +} + export default function IpfsNode() { - const ipfsRepository = useApi.ipfsClientTargetGet(); - const [ipfsClientTarget, setIpfsClientTarget] = useState(null); - const [ipfsGatewayTarget, setIpfsGatewayTarget] = useState(null); + const ipfsRepository = useApi.ipfsGatewayUrlsGet(); + // Store as comma-separated string for easier editing + const [ipfsGatewayUrlsStr, setIpfsGatewayUrlsStr] = useState(""); useEffect(() => { - if (ipfsRepository.data) setIpfsClientTarget(ipfsRepository.data.ipfsClientTarget); - if (ipfsRepository.data) setIpfsGatewayTarget(ipfsRepository.data.ipfsGateway); + if (ipfsRepository.data) { + setIpfsGatewayUrlsStr(urlsToString(ipfsRepository.data.ipfsGatewayUrls)); + } }, [ipfsRepository.data]); - async function changeIpfsClient() { - if (ipfsClientTarget && ipfsGatewayTarget) + async function saveGatewayUrls() { + if (ipfsGatewayUrlsStr) { + const gatewayUrls = stringToUrls(ipfsGatewayUrlsStr); + if (gatewayUrls.length === 0) { + // Don't allow empty gateway URLs + return; + } + await withToast( () => - api.ipfsClientTargetSet({ - ipfsRepository: { - ipfsClientTarget: ipfsClientTarget, - ipfsGateway: ipfsGatewayTarget - } + api.ipfsGatewayUrlsSet({ + ipfsGatewayUrls: gatewayUrls }), { - message: `Setting IPFS mode ${ipfsClientTarget}...`, - onSuccess: `Successfully changed to ${ipfsClientTarget}` + message: "Saving IPFS gateway URLs...", + onSuccess: "Successfully updated IPFS gateway URLs" } ); + } await ipfsRepository.revalidate(); } + /** + * Check if settings have changed from saved values + */ + function hasChanges(): boolean { + if (!ipfsRepository.data) return false; + + const savedUrls = ipfsRepository.data.ipfsGatewayUrls; + const currentUrls = stringToUrls(ipfsGatewayUrlsStr); + + return JSON.stringify(savedUrls) !== JSON.stringify(currentUrls); + } + return (
- IPFS Node + IPFS Gateway Configuration
- Dappnode uses IPFS to distribute Dappnode packages in a decentralized way. Choose to connect to a remote - IPFS gateway or use your own local IPFS node. + Dappnode uses IPFS to distribute packages in a decentralized way. Configure your IPFS gateway URLs below.
- More information at: How to use Dappnode IPFS remote + Resilient fetching: The system will try your local IPFS node first (if available), then + race all configured gateways simultaneously - the first to respond wins.
- - {ipfsRepository.data ? ( - <> - -
+ {ipfsRepository.data ? ( + <> + + +
-
- -
- - ) : null} +
+ +
+ + ) : null} +
+ More information at: How to use Dappnode IPFS remote +
+
); diff --git a/packages/daemons/src/repositoryHealth/index.ts b/packages/daemons/src/repositoryHealth/index.ts index 4c2c34477..5db8c7635 100644 --- a/packages/daemons/src/repositoryHealth/index.ts +++ b/packages/daemons/src/repositoryHealth/index.ts @@ -2,7 +2,6 @@ import { logs } from "@dappnode/logger"; import { runAtMostEvery } from "@dappnode/utils"; import { notifications } from "@dappnode/notifications"; import { Category, Priority, Status } from "@dappnode/types"; -import * as db from "@dappnode/db"; import { getIpfsUrl } from "@dappnode/installer"; import { params } from "@dappnode/params"; import { eventBus } from "@dappnode/eventbus"; @@ -15,7 +14,6 @@ let ethFailureCount = 0; let ethNotificationSent = false; async function checkIpfsHealth(): Promise { - const ipfsClientTarget = db.ipfsClientTarget.get(); const ipfsUrl = getIpfsUrl(); const controller = new AbortController(); @@ -36,7 +34,7 @@ async function checkIpfsHealth(): Promise { if (!res.ok) throw new Error(`Status ${res.status}`); - logs.info(`IPFS endpoint (${ipfsClientTarget}) at ${ipfsUrl} is healthy`); + logs.info(`IPFS endpoint at ${ipfsUrl} is healthy`); // reset failure count on success ipfsFailureCount = 0; @@ -56,7 +54,7 @@ async function checkIpfsHealth(): Promise { } } catch (error) { clearTimeout(timeout); - logs.error(`IPFS endpoint (${ipfsClientTarget}) at ${ipfsUrl} is unhealthy: ${error}`); + logs.error(`IPFS endpoint at ${ipfsUrl} is unhealthy: ${error}`); // increment failure count and send notification after threshold ipfsFailureCount += 1; @@ -64,12 +62,12 @@ async function checkIpfsHealth(): Promise { await notifications.sendNotification({ title: "Your Dappnode IPFS endpoint is not resolving content correctly.", dnpName: params.dappmanagerDnpName, - body: `Dappnode IPFS endpoint (${ipfsClientTarget}) at ${ipfsUrl} is currently unreachable or not resolving content correctly. This may affect access to decentralized content or applications relying on IPFS.`, + body: `Dappnode IPFS endpoint at ${ipfsUrl} is currently unreachable or not resolving content correctly. This may affect access to decentralized content or applications relying on IPFS.`, category: Category.system, priority: Priority.high, status: Status.triggered, callToAction: { - title: `Switch to ${ipfsClientTarget && ipfsClientTarget === "local" ? "Remote" : "Local"} IPFS`, + title: "Configure IPFS Gateways", url: "http://my.dappnode/system/ipfs" }, isBanner: true, diff --git a/packages/dappmanager/src/api/routes/metrics.ts b/packages/dappmanager/src/api/routes/metrics.ts index 5d2262fc2..f5f4a8e60 100644 --- a/packages/dappmanager/src/api/routes/metrics.ts +++ b/packages/dappmanager/src/api/routes/metrics.ts @@ -28,19 +28,15 @@ export const metrics = wrapHandler(async (_, res) => { // Create a Registry which registers the metrics const register = new client.Registry(); -// IPFS node local or remote +// IPFS gateway URLs configured count register.registerMetric( new client.Gauge({ - name: "dappmanager_ipfs_client_target_local", - help: "Ipfs client target local", - labelNames: ["ipfsClientTargetLocal"], + name: "dappmanager_ipfs_gateway_count", + help: "Number of IPFS gateways configured", + labelNames: ["gatewayCount"], collect() { - const ipfsClientTarget = db.ipfsClientTarget.get(); - if (ipfsClientTarget === "local") { - this.set({ ipfsClientTargetLocal: "local" }, 1); - } else { - this.set({ ipfsClientTargetLocal: "local" }, 0); - } + const ipfsGatewayUrls = db.ipfsGatewayUrls.get(); + this.set({ gatewayCount: "count" }, ipfsGatewayUrls.length); } }) ); diff --git a/packages/dappmanager/src/calls/index.ts b/packages/dappmanager/src/calls/index.ts index 085209eeb..9a76fdfa0 100644 --- a/packages/dappmanager/src/calls/index.ts +++ b/packages/dappmanager/src/calls/index.ts @@ -43,8 +43,8 @@ export { } from "./notifications.js"; export * from "./httpsPortal.js"; export { ipfsTest } from "./ipfsTest.js"; -export { ipfsClientTargetSet } from "./ipfsClientTargetSet.js"; -export { ipfsClientTargetGet } from "./ipfsClientTargetGet.js"; +export { ipfsGatewayUrlsSet } from "./ipfsGatewayUrlsSet.js"; +export { ipfsGatewayUrlsGet } from "./ipfsGatewayUrlsGet.js"; export { ipPublicGet } from "./ipPublicGet.js"; export * from "./localProxy.js"; export * from "./manageLvm.js"; diff --git a/packages/dappmanager/src/calls/ipfsClientTargetGet.ts b/packages/dappmanager/src/calls/ipfsClientTargetGet.ts index b355643f0..0982e9efa 100644 --- a/packages/dappmanager/src/calls/ipfsClientTargetGet.ts +++ b/packages/dappmanager/src/calls/ipfsClientTargetGet.ts @@ -1,9 +1,8 @@ import * as db from "@dappnode/db"; import { IpfsRepository } from "@dappnode/types"; -export async function ipfsClientTargetGet(): Promise { +export async function ipfsGatewayUrlsGet(): Promise { return { - ipfsClientTarget: db.ipfsClientTarget.get(), - ipfsGateway: db.ipfsGateway.get() + ipfsGatewayUrls: db.ipfsGatewayUrls.get() }; } diff --git a/packages/dappmanager/src/calls/ipfsClientTargetSet.ts b/packages/dappmanager/src/calls/ipfsClientTargetSet.ts index f900ce0bb..4e9e04767 100644 --- a/packages/dappmanager/src/calls/ipfsClientTargetSet.ts +++ b/packages/dappmanager/src/calls/ipfsClientTargetSet.ts @@ -1,47 +1,41 @@ -import { IpfsRepository, IpfsClientTarget } from "@dappnode/types"; import { params } from "@dappnode/params"; import * as db from "@dappnode/db"; import { dappnodeInstaller } from "../index.js"; import { eventBus } from "@dappnode/eventbus"; /** - * Changes the IPFS client + * Sets the IPFS gateway URLs for package fetching */ -export async function ipfsClientTargetSet({ ipfsRepository }: { ipfsRepository: IpfsRepository }): Promise { - if (!ipfsRepository.ipfsClientTarget) throw Error(`Argument target must be defined`); +export async function ipfsGatewayUrlsSet({ ipfsGatewayUrls }: { ipfsGatewayUrls: string[] }): Promise { + if (!ipfsGatewayUrls || ipfsGatewayUrls.length === 0) { + throw Error(`At least one gateway URL must be provided`); + } - await changeIpfsClient(ipfsRepository.ipfsClientTarget, ipfsRepository.ipfsGateway); + await changeIpfsGatewayUrls(ipfsGatewayUrls); // Emit event to trigger notifier healthcheck notification eventBus.ipfsRepositoryChanged.emit(); } /** - * Changes IPFS client from remote to local and viceversa. - * I local mode is set and IPFS is not installed, it will install - * the IPFS package - * @param nextTarget "local" | "remote" - * @param nextGateway Gateway endpoint to be used by remote node. By default dappnode gateway + * Changes IPFS gateway URLs used for package fetching + * @param nextGatewayUrls Gateway endpoints to be used */ -async function changeIpfsClient(nextTarget: IpfsClientTarget, nextGateway?: string): Promise { +async function changeIpfsGatewayUrls(nextGatewayUrls: string[]): Promise { try { - // Return if targets and gateways are equal - const currentTarget = db.ipfsClientTarget.get(); - const currentGateway = db.ipfsGateway.get(); - if (currentTarget === nextTarget && currentGateway === nextGateway) return; + // Return if gateway URLs are equal + const currentGatewayUrls = db.ipfsGatewayUrls.get(); + if (JSON.stringify(currentGatewayUrls) === JSON.stringify(nextGatewayUrls)) { + return; + } - if (nextTarget === IpfsClientTarget.local) { - db.ipfsClientTarget.set(IpfsClientTarget.local); - dappnodeInstaller.changeIpfsGatewayUrl(params.IPFS_LOCAL); - } else { - // Set new values in db - db.ipfsGateway.set(nextGateway || params.IPFS_GATEWAY); - db.ipfsClientTarget.set(IpfsClientTarget.remote); + // Set new values in db + const gatewayUrls = nextGatewayUrls.length > 0 ? nextGatewayUrls : params.IPFS_REMOTE_URLS; + db.ipfsGatewayUrls.set(gatewayUrls); - // Change IPFS host - dappnodeInstaller.changeIpfsGatewayUrl(db.ipfsGateway.get()); - } + // Change IPFS gateway URLs in the installer + dappnodeInstaller.changeIpfsGatewayUrls(db.ipfsGatewayUrls.get()); } catch (e) { - throw Error(`Error changing ipfs client to ${nextTarget}, ${e}`); + throw Error(`Error changing IPFS gateway URLs, ${e}`); } } diff --git a/packages/dappmanager/src/calls/ipfsGatewayUrlsGet.ts b/packages/dappmanager/src/calls/ipfsGatewayUrlsGet.ts new file mode 100644 index 000000000..0982e9efa --- /dev/null +++ b/packages/dappmanager/src/calls/ipfsGatewayUrlsGet.ts @@ -0,0 +1,8 @@ +import * as db from "@dappnode/db"; +import { IpfsRepository } from "@dappnode/types"; + +export async function ipfsGatewayUrlsGet(): Promise { + return { + ipfsGatewayUrls: db.ipfsGatewayUrls.get() + }; +} diff --git a/packages/dappmanager/src/calls/ipfsGatewayUrlsSet.ts b/packages/dappmanager/src/calls/ipfsGatewayUrlsSet.ts new file mode 100644 index 000000000..4e9e04767 --- /dev/null +++ b/packages/dappmanager/src/calls/ipfsGatewayUrlsSet.ts @@ -0,0 +1,41 @@ +import { params } from "@dappnode/params"; +import * as db from "@dappnode/db"; +import { dappnodeInstaller } from "../index.js"; +import { eventBus } from "@dappnode/eventbus"; + +/** + * Sets the IPFS gateway URLs for package fetching + */ +export async function ipfsGatewayUrlsSet({ ipfsGatewayUrls }: { ipfsGatewayUrls: string[] }): Promise { + if (!ipfsGatewayUrls || ipfsGatewayUrls.length === 0) { + throw Error(`At least one gateway URL must be provided`); + } + + await changeIpfsGatewayUrls(ipfsGatewayUrls); + + // Emit event to trigger notifier healthcheck notification + eventBus.ipfsRepositoryChanged.emit(); +} + +/** + * Changes IPFS gateway URLs used for package fetching + * @param nextGatewayUrls Gateway endpoints to be used + */ +async function changeIpfsGatewayUrls(nextGatewayUrls: string[]): Promise { + try { + // Return if gateway URLs are equal + const currentGatewayUrls = db.ipfsGatewayUrls.get(); + if (JSON.stringify(currentGatewayUrls) === JSON.stringify(nextGatewayUrls)) { + return; + } + + // Set new values in db + const gatewayUrls = nextGatewayUrls.length > 0 ? nextGatewayUrls : params.IPFS_REMOTE_URLS; + db.ipfsGatewayUrls.set(gatewayUrls); + + // Change IPFS gateway URLs in the installer + dappnodeInstaller.changeIpfsGatewayUrls(db.ipfsGatewayUrls.get()); + } catch (e) { + throw Error(`Error changing IPFS gateway URLs, ${e}`); + } +} diff --git a/packages/dappmanager/src/initializeDb.ts b/packages/dappmanager/src/initializeDb.ts index dd747e44e..18b458430 100644 --- a/packages/dappmanager/src/initializeDb.ts +++ b/packages/dappmanager/src/initializeDb.ts @@ -6,7 +6,7 @@ import { getInternalIp, getServerName, getStaticIp, ping } from "./utils/index.j import { getExternalUpnpIp, isUpnpAvailable } from "@dappnode/upnpc"; import { writeGlobalEnvsToEnvFile } from "@dappnode/db"; import { params } from "@dappnode/params"; -import { IdentityInterface, IpfsClientTarget } from "@dappnode/types"; +import { IdentityInterface } from "@dappnode/types"; import { logs } from "@dappnode/logger"; import { localProxyingEnableDisable } from "./calls/index.js"; import { pause, shell, getPublicIpFromUrls } from "@dappnode/utils"; @@ -35,25 +35,31 @@ function returnNullIfError(fn: () => Promise, silent?: boolean): () => P */ export async function initializeDb(): Promise { /** - * ipfsClientTarget + * Migrate from single ipfsGateway (string) to ipfsGatewayUrls (string[]) + * This handles the transition from the old single-gateway format to multi-gateway support */ try { - const ipfsClientTarget = db.ipfsClientTarget.get(); - if (!ipfsClientTarget) { - logs.info("ipfsClientTarget not found, setting to local"); - db.ipfsClientTarget.set(IpfsClientTarget.local); + const legacyGateway = db.ipfsGatewayLegacy.get(); + const currentUrls = db.ipfsGatewayUrls.get(); + + // Check if we need to migrate: legacy value exists and is different from defaults + if (legacyGateway && legacyGateway !== params.IPFS_REMOTE_URLS[0]) { + // Migrate deprecated endpoint + const migratedGateway = + legacyGateway === "http://ipfs.dappnode.io:8081" ? params.IPFS_REMOTE_URLS[0] : legacyGateway; + + // If currentUrls is still the default, prepend the migrated legacy gateway + if (JSON.stringify(currentUrls) === JSON.stringify(params.IPFS_REMOTE_URLS)) { + // Use legacy gateway as primary, keep defaults as fallbacks + const newUrls = [migratedGateway, ...params.IPFS_REMOTE_URLS.filter((url) => url !== migratedGateway)]; + db.ipfsGatewayUrls.set(newUrls); + logs.info(`Migrated IPFS gateway from ${legacyGateway} to multi-gateway format`); + } } } catch (e) { - logs.error("Error getting ipfsClientTarget", e); - db.ipfsClientTarget.set(IpfsClientTarget.local); + logs.warn("Error migrating IPFS gateway settings", e); } - /** - * Migrate ipfs remote gateway endpoint from http://ipfs.dappnode.io:8081 to https://ipfs.gateway.dappnode.io - * The endpoint http://ipfs.dappnode.io:8081 is being deprecated - */ - if (db.ipfsGateway.get() === "http://ipfs.dappnode.io:8081") db.ipfsGateway.set(params.IPFS_REMOTE); - /** * * diff --git a/packages/db/src/ipfsClient.ts b/packages/db/src/ipfsClient.ts index 5912ff2ba..f11043aba 100644 --- a/packages/db/src/ipfsClient.ts +++ b/packages/db/src/ipfsClient.ts @@ -1,11 +1,12 @@ import { dbMain } from "./dbFactory.js"; -import { IpfsClientTarget } from "@dappnode/types"; import { params } from "@dappnode/params"; // User chosen properties -const IPFS_CLIENT_TARGET = "ipfs-client-target"; -const IPFS_GATEWAY = "ipfs-gateway"; +const IPFS_GATEWAY_URLS = "ipfs-gateway-urls"; +// Legacy key for migration +const IPFS_GATEWAY_LEGACY = "ipfs-gateway"; -export const ipfsClientTarget = dbMain.staticKey(IPFS_CLIENT_TARGET, IpfsClientTarget.local); +export const ipfsGatewayUrls = dbMain.staticKey(IPFS_GATEWAY_URLS, params.IPFS_REMOTE_URLS); -export const ipfsGateway = dbMain.staticKey(IPFS_GATEWAY, params.IPFS_REMOTE); +// Legacy accessor for migration purposes only +export const ipfsGatewayLegacy = dbMain.staticKey(IPFS_GATEWAY_LEGACY, params.IPFS_REMOTE_URLS[0]); diff --git a/packages/installer/src/dappnodeInstaller.ts b/packages/installer/src/dappnodeInstaller.ts index 219875f42..922e40aa7 100644 --- a/packages/installer/src/dappnodeInstaller.ts +++ b/packages/installer/src/dappnodeInstaller.ts @@ -3,7 +3,6 @@ import { DappnodeRepository } from "@dappnode/toolkit"; import * as db from "@dappnode/db"; import { DistributedFile, - IpfsClientTarget, PackageRelease, ManifestWithImage, Compose, @@ -30,18 +29,24 @@ import { omit } from "lodash-es"; import { JsonRpcApiProvider } from "ethers"; /** + * Returns the IPFS gateway URLs to initialize the ipfs instance. + * These URLs are used for resilient fetching - local node is tried first, then gateways are raced. + */ +export function getIpfsGatewayUrls(): string[] { + // For testing + if (params.IPFS_HOST) return [params.IPFS_HOST]; + + // Return configured gateway URLs from db + return db.ipfsGatewayUrls.get(); +} + +/** + * @deprecated Use getIpfsGatewayUrls() instead * Returns the ipfsUrl to initialize the ipfs instance */ export function getIpfsUrl(): string { - // Fort testing - if (params.IPFS_HOST) return params.IPFS_HOST; - - const ipfsClientTarget = db.ipfsClientTarget.get(); - if (!ipfsClientTarget) throw Error("Ipfs client target is not set"); - // local - if (ipfsClientTarget === IpfsClientTarget.local) return params.IPFS_LOCAL; - // remote - return db.ipfsGateway.get(); + const urls = getIpfsGatewayUrls(); + return urls[0]; } export class DappnodeInstaller extends DappnodeRepository { @@ -50,9 +55,9 @@ export class DappnodeInstaller extends DappnodeRepository { } private async updateProviders(): Promise { - const newIpfsUrl = getIpfsUrl(); + const newIpfsUrls = getIpfsGatewayUrls(); // super.changeEthProvider(); - super.changeIpfsGatewayUrl(newIpfsUrl); + super.changeIpfsGatewayUrls(newIpfsUrls); } /** diff --git a/packages/params/src/params.ts b/packages/params/src/params.ts index 1520c55ac..b365976c7 100644 --- a/packages/params/src/params.ts +++ b/packages/params/src/params.ts @@ -154,7 +154,7 @@ export const params = { AUTO_UPDATE_INCLUDE_IPFS_VERSIONS: false, // Install method parameters - ALWAYS_DAPPGETBASIC: process.env.ALWAYS_DAPPGETBASIC === 'true', + ALWAYS_DAPPGETBASIC: process.env.ALWAYS_DAPPGETBASIC === "true", // Watchers TEMPERATURE_DAEMON_INTERVAL: 5 * MINUTE, AUTO_UPDATE_DAEMON_INTERVAL: 30 * MINUTE, @@ -167,7 +167,16 @@ export const params = { IPFS_HOST: process.env.IPFS_HOST || process.env.IPFS_REDIRECT, IPFS_TIMEOUT: 0.5 * MINUTE, IPFS_LOCAL: "http://ipfs.dappnode:8080", + IPFS_LOCAL_API: "http://ipfs.dappnode:5001", IPFS_REMOTE: "https://ipfs-gateway.dappnode.net", + /** Default list of IPFS gateway URLs to use for resilient content fetching */ + IPFS_REMOTE_URLS: [ + "https://ipfs-gateway.dappnode.net", + "https://ipfs-gateway-dev.dappnode.net", + "https://gateway.pinata.cloud", + "https://cloudflare-ipfs.com", + "https://dweb.link" + ], // Web3 parameters ETH_MAINNET_RPC_URL_REMOTE: process.env.ETH_MAINNET_RPC_URL_REMOTE || "https://web3.dappnode.net", diff --git a/packages/toolkit/src/repository/repository.ts b/packages/toolkit/src/repository/repository.ts index b0fe9a2be..4087ff4c5 100644 --- a/packages/toolkit/src/repository/repository.ts +++ b/packages/toolkit/src/repository/repository.ts @@ -32,39 +32,61 @@ import { JsonRpcApiProvider } from "ethers"; const source = "ipfs" as const; +/** Timeout for local IPFS node requests in milliseconds */ +const LOCAL_IPFS_TIMEOUT_MS = 30_000; + /** * The DappnodeRepository class extends ApmRepository class to provide methods to interact with the IPFS network. - * To fetch IPFS content it uses dag endpoint for CAR content validation + * To fetch IPFS content it uses dag endpoint for CAR content validation. + * + * Implements a resilient IPFS fetching strategy: + * 1. Try local IPFS node first (via API port 5001) + * 2. If local fails, race all configured gateway URLs simultaneously + * 3. First successful response wins, others are aborted * * @extends ApmRepository */ export class DappnodeRepository extends ApmRepository { - protected gatewayUrl: string; - protected localIpfsUrl = "http://ipfs.dappnode:5001"; + /** Array of IPFS gateway URLs to use for fetching content */ + protected gatewayUrls: string[]; + /** Local IPFS node API URL (port 5001) for RPC operations */ + protected localIpfsApiUrl = "http://ipfs.dappnode:5001"; + /** Local IPFS node gateway URL (port 8080) for CAR retrieval */ + protected localIpfsGatewayUrl = "http://ipfs.dappnode:8080"; /** * Constructs an instance of DappnodeRepository - * @param ipfsUrl - The URL of the IPFS network node. - * @param ethUrl - The URL of the Ethereum node to connect to. + * @param ipfsUrl - The primary URL of the IPFS network node (for backward compatibility). + * @param provider - The Ethereum JSON-RPC provider. */ constructor(ipfsUrl: string, provider: JsonRpcApiProvider) { super(provider); - this.gatewayUrl = ipfsUrl.replace(/\/?$/, ""); // e.g. "https://gateway.pinata.cloud" + // Initialize with single URL for backward compatibility + this.gatewayUrls = [ipfsUrl.replace(/\/?$/, "")]; + } + + /** + * Changes the IPFS gateway URLs for multi-gateway resilient fetching. + * @param ipfsUrls - Array of IPFS gateway URLs. + */ + public changeIpfsGatewayUrls(ipfsUrls: string[]): void { + this.gatewayUrls = ipfsUrls.map((url) => url.replace(/\/?$/, "")); } /** - * Changes the IPFS provider and target. + * @deprecated Use changeIpfsGatewayUrls instead for multi-gateway support + * Changes the IPFS provider and target (backward compatibility). * @param ipfsUrl - The new URL of the IPFS network node. */ public changeIpfsGatewayUrl(ipfsUrl: string): void { - this.gatewayUrl = ipfsUrl.replace(/\/?$/, ""); + this.gatewayUrls = [ipfsUrl.replace(/\/?$/, "")]; } /** * Pin content to local IPFS node */ public async pinAddLocal(hash: string): Promise { - await fetch(`${this.localIpfsUrl}/api/v0/pin/add?arg=${hash}`, { + await fetch(`${this.localIpfsApiUrl}/api/v0/pin/add?arg=${hash}`, { method: "POST", headers: { "Content-Type": "application/json" @@ -76,7 +98,7 @@ export class DappnodeRepository extends ApmRepository { * Unpin content from local IPFS node */ public async pinRmLocal(hash: string): Promise { - await fetch(`${this.localIpfsUrl}/api/v0/pin/rm?arg=${hash}`, { + await fetch(`${this.localIpfsApiUrl}/api/v0/pin/rm?arg=${hash}`, { method: "POST", headers: { "Content-Type": "application/json" @@ -250,6 +272,8 @@ export class DappnodeRepository extends ApmRepository { * Downloads the content pointed by the given hash, parses it to UTF8 and returns it as a string. * This function is intended for small files. * + * Uses resilient fetching: tries local IPFS first, then races all gateway URLs. + * * @param hash - The content identifier (CID) of the file to download. * @param maxLength - The maximum length of the file in bytes. If the downloaded file exceeds this length, an error is thrown. * @returns The downloaded file content as a UTF8 string. @@ -259,7 +283,7 @@ export class DappnodeRepository extends ApmRepository { */ public async writeFileToMemory(hash: string, maxLength?: number): Promise { const chunks: Uint8Array[] = []; - const { carReader, root } = await this.getAndVerifyContentFromGateway(hash); + const { carReader, root } = await this.getAndVerifyContentResilient(hash); const content = await this.unpackCarReader(carReader, root); for await (const chunk of content) chunks.push(chunk); @@ -285,6 +309,8 @@ export class DappnodeRepository extends ApmRepository { * Downloads the content pointed by the given hash and writes it directly to the filesystem. * This function is intended for large files, such as Docker images. * + * Uses resilient fetching: tries local IPFS first, then races all gateway URLs. + * * IMPORTANT: This function is not supported in the browser. * * @param args - The arguments object. @@ -309,7 +335,7 @@ export class DappnodeRepository extends ApmRepository { fileSize?: number; progress?: (n: number) => void; }): Promise { - const { carReader, root } = await this.getAndVerifyContentFromGateway(hash); + const { carReader, root } = await this.getAndVerifyContentResilient(hash); const readable = await this.unpackCarReader(carReader, root); return new Promise((resolve, reject) => { @@ -373,7 +399,9 @@ export class DappnodeRepository extends ApmRepository { /** * Lists the contents of a directory pointed by the given hash. - * ipfs.dag.get => reutrns `Tsize`! + * ipfs.dag.get => returns `Tsize`! + * + * Uses resilient fetching: tries local IPFS first, then races all gateway URLs. * * TODO: research why the size is different, i.e for the hash QmWcJrobqhHF7GWpqEbxdv2cWCCXbACmq85Hh7aJ1eu8rn Tsize is 64461521 and size is 64446140 * @@ -383,21 +411,9 @@ export class DappnodeRepository extends ApmRepository { */ public async list(hash: string): Promise { const cidStr = this.sanitizeIpfsPath(hash.toString()); - const url = `${this.gatewayUrl}/ipfs/${cidStr}?format=dag-json`; - const res = await fetch(url, { - headers: { Accept: "application/vnd.ipld.dag-json" } - }); - if (!res.ok) { - throw new Error(`Failed to list directory ${cidStr}: ${res.status} ${res.statusText}`); - } - const dagJson = (await res.json()) as { - Links?: Array<{ - Name: string; - Hash: { "/": string }; - Tsize: number; - }>; - }; + // Resilient fetching: try local first, then race gateways + const dagJson = await this.fetchDagJsonResilient(cidStr); if (!dagJson.Links) { throw new Error(`Invalid IPFS directory CID ${cidStr}`); @@ -413,29 +429,232 @@ export class DappnodeRepository extends ApmRepository { } /** - * Gets the content from an IPFS gateway using the given hash and verifies its integrity. - * The content is returned as a CAR reader and the root CID. + * Fetches DAG-JSON from IPFS using resilient strategy. + * Tries local IPFS first, then races all gateway URLs. + */ + private async fetchDagJsonResilient(cidStr: string): Promise<{ + Links?: Array<{ + Name: string; + Hash: { "/": string }; + Tsize: number; + }>; + }> { + const errors: Array<{ source: string; error: unknown }> = []; + + // 1. Try local IPFS gateway first (with timeout) + const localAbortController = new AbortController(); + const localTimeout = setTimeout(() => localAbortController.abort(), LOCAL_IPFS_TIMEOUT_MS); + try { + const localResult = await this.fetchDagJsonFromUrl(this.localIpfsGatewayUrl, cidStr, localAbortController.signal); + clearTimeout(localTimeout); + console.log(`IPFS resolved ${cidStr} via local node (${this.localIpfsGatewayUrl})`); + return localResult; + } catch (err) { + clearTimeout(localTimeout); + const errorMsg = (err as Error).name === "AbortError" ? "Timeout after 30s" : (err as Error).message; + errors.push({ source: "local", error: errorMsg }); + } + + // 2. Race all remote gateways simultaneously + if (this.gatewayUrls.length === 0) { + throw new Error(`No IPFS gateways configured. Errors: ${this.formatErrors(errors)}`); + } + + const raceResult = await this.raceGatewaysForDagJson(cidStr, errors); + if (raceResult) return raceResult; + + throw new Error(`All IPFS sources failed for ${cidStr}. Errors: ${this.formatErrors(errors)}`); + } + + /** + * Fetches DAG-JSON from a specific gateway URL. + */ + private async fetchDagJsonFromUrl( + gatewayUrl: string, + cidStr: string, + signal?: AbortSignal + ): Promise<{ + Links?: Array<{ + Name: string; + Hash: { "/": string }; + Tsize: number; + }>; + }> { + const url = `${gatewayUrl}/ipfs/${cidStr}?format=dag-json`; + const res = await fetch(url, { + headers: { Accept: "application/vnd.ipld.dag-json" }, + signal + }); + if (!res.ok) { + throw new Error(`Failed to list directory ${cidStr}: ${res.status} ${res.statusText}`); + } + return (await res.json()) as { + Links?: Array<{ + Name: string; + Hash: { "/": string }; + Tsize: number; + }>; + }; + } + + /** + * Races all configured gateway URLs for DAG-JSON content. + * Returns the first successful result, aborts others. + */ + private async raceGatewaysForDagJson( + cidStr: string, + errors: Array<{ source: string; error: unknown }> + ): Promise<{ + Links?: Array<{ + Name: string; + Hash: { "/": string }; + Tsize: number; + }>; + } | null> { + const abortControllers = this.gatewayUrls.map(() => new AbortController()); + + const promises = this.gatewayUrls.map(async (gatewayUrl, index) => { + try { + const url = `${gatewayUrl}/ipfs/${cidStr}?format=dag-json`; + const res = await fetch(url, { + headers: { Accept: "application/vnd.ipld.dag-json" }, + signal: abortControllers[index].signal + }); + if (!res.ok) { + throw new Error(`HTTP ${res.status} ${res.statusText}`); + } + const data = await res.json(); + // Success! Abort all other requests + abortControllers.forEach((ctrl, i) => { + if (i !== index) ctrl.abort(); + }); + console.log(`IPFS resolved ${cidStr} via gateway (${gatewayUrl})`); + return { success: true as const, data, gatewayUrl }; + } catch (err) { + if ((err as Error).name !== "AbortError") { + errors.push({ source: gatewayUrl, error: err }); + } + return { success: false as const, gatewayUrl }; + } + }); + + const results = await Promise.all(promises); + const successResult = results.find((r) => r.success); + return successResult?.success ? successResult.data : null; + } + + /** + * Resilient content fetching: tries local IPFS first, then races all gateway URLs. + * Gets the content as a verified CAR reader. * * @param hash - The content identifier (CID) of the content to get and verify. * @returns The content as a CAR reader and the root CID. - * @throws Error when the root CID does not match the provided hash (content is untrusted). + * @throws Error when all sources fail or content is untrusted. */ - private async getAndVerifyContentFromGateway(hash: string): Promise<{ + private async getAndVerifyContentResilient(hash: string): Promise<{ carReader: CarReader; root: CID; }> { - // 1. Download the CAR - const url = `${this.gatewayUrl}/ipfs/${hash}?format=car`; + const errors: Array<{ source: string; error: unknown }> = []; + + // 1. Try local IPFS gateway first (with timeout) + const localAbortController = new AbortController(); + const localTimeout = setTimeout(() => localAbortController.abort(), LOCAL_IPFS_TIMEOUT_MS); + try { + const localResult = await this.getAndVerifyContentFromUrl( + this.localIpfsGatewayUrl, + hash, + localAbortController.signal + ); + clearTimeout(localTimeout); + console.log(`IPFS resolved ${hash} via local node (${this.localIpfsGatewayUrl})`); + return localResult; + } catch (err) { + clearTimeout(localTimeout); + const errorMsg = (err as Error).name === "AbortError" ? "Timeout after 30s" : (err as Error).message; + errors.push({ source: "local", error: errorMsg }); + } + + // 2. Race all remote gateways simultaneously + if (this.gatewayUrls.length === 0) { + throw new Error(`No IPFS gateways configured. Errors: ${this.formatErrors(errors)}`); + } + + const raceResult = await this.raceGatewaysForCar(hash, errors); + if (raceResult) return raceResult; + + throw new Error(`All IPFS sources failed for ${hash}. Errors: ${this.formatErrors(errors)}`); + } + + /** + * Races all configured gateway URLs for CAR content. + * Returns the first successful verified result, aborts others. + */ + private async raceGatewaysForCar( + hash: string, + errors: Array<{ source: string; error: unknown }> + ): Promise<{ carReader: CarReader; root: CID } | null> { + const abortControllers = this.gatewayUrls.map(() => new AbortController()); + + const promises = this.gatewayUrls.map(async (gatewayUrl, index) => { + try { + const url = `${gatewayUrl}/ipfs/${hash}?format=car`; + const res = await fetch(url, { + headers: { Accept: "application/vnd.ipld.car" }, + signal: abortControllers[index].signal + }); + if (!res.ok) { + throw new Error(`HTTP ${res.status} ${res.statusText}`); + } + + const bytes = new Uint8Array(await res.arrayBuffer()); + const carReader = await CarReader.fromBytes(bytes); + + // Verify the root CID + const roots = await carReader.getRoots(); + const root = roots[0]; + if (roots.length !== 1 || root.toString() !== CID.parse(hash).toString()) { + throw new Error(`UNTRUSTED CONTENT: expected root ${hash}, got ${roots}`); + } + + // Success! Abort all other requests + abortControllers.forEach((ctrl, i) => { + if (i !== index) ctrl.abort(); + }); + + console.log(`IPFS resolved ${hash} via gateway (${gatewayUrl})`); + return { success: true as const, data: { carReader, root }, gatewayUrl }; + } catch (err) { + if ((err as Error).name !== "AbortError") { + errors.push({ source: gatewayUrl, error: err }); + } + return { success: false as const, gatewayUrl }; + } + }); + + const results = await Promise.all(promises); + const successResult = results.find((r) => r.success); + return successResult?.success ? successResult.data : null; + } + + /** + * Gets and verifies content from a specific gateway URL. + */ + private async getAndVerifyContentFromUrl( + gatewayUrl: string, + hash: string, + signal?: AbortSignal + ): Promise<{ carReader: CarReader; root: CID }> { + const url = `${gatewayUrl}/ipfs/${hash}?format=car`; const res = await fetch(url, { - headers: { Accept: "application/vnd.ipld.car" } + headers: { Accept: "application/vnd.ipld.car" }, + signal }); if (!res.ok) throw new Error(`Gateway error: ${res.status} ${res.statusText}`); - // 2. Parse into a CarReader const bytes = new Uint8Array(await res.arrayBuffer()); const carReader = await CarReader.fromBytes(bytes); - // 3. Verify the root CID const roots = await carReader.getRoots(); const root = roots[0]; if (roots.length !== 1 || root.toString() !== CID.parse(hash).toString()) { @@ -445,6 +664,13 @@ export class DappnodeRepository extends ApmRepository { return { carReader, root }; } + /** + * Formats error array for logging. + */ + private formatErrors(errors: Array<{ source: string; error: unknown }>): string { + return errors.map((e) => `${e.source}: ${e.error instanceof Error ? e.error.message : String(e.error)}`).join("; "); + } + /** * Unpacks a CAR reader and returns an async iterable of uint8arrays. * diff --git a/packages/types/src/calls.ts b/packages/types/src/calls.ts index fb7b70102..fee4cb4be 100644 --- a/packages/types/src/calls.ts +++ b/packages/types/src/calls.ts @@ -929,13 +929,7 @@ export interface DistributedFile { } export interface IpfsRepository { - ipfsClientTarget: IpfsClientTarget; - ipfsGateway: string; -} - -export enum IpfsClientTarget { - local = "local", - remote = "remote" + ipfsGatewayUrls: string[]; } /** diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index ab245403c..1d14336d3 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -387,14 +387,14 @@ export interface Routes { ipfsTest(): Promise; /** - * Sets the ipfs client target: local | remote + * Sets the IPFS gateway URLs for package fetching */ - ipfsClientTargetSet(kwargs: { ipfsRepository: IpfsRepository }): Promise; + ipfsGatewayUrlsSet(kwargs: { ipfsGatewayUrls: string[] }): Promise; /** - * Gets the Ipfs client target + * Gets the IPFS gateway URLs for package fetching */ - ipfsClientTargetGet(): Promise; + ipfsGatewayUrlsGet(): Promise; /** * Returns the keystores imported for the given networks. @@ -902,8 +902,8 @@ export const routesData: { [P in keyof Routes]: RouteData } = { httpsPortalMappingsRecreate: {}, httpsPortalExposableServicesGet: {}, ipfsTest: {}, - ipfsClientTargetSet: {}, - ipfsClientTargetGet: {}, + ipfsGatewayUrlsSet: {}, + ipfsGatewayUrlsGet: {}, keystoresGetByNetwork: { log: true }, localProxyingEnableDisable: { log: true }, localProxyingStatusGet: {},