Skip on first sub-dependency promise reject

packages/installer/src/dappGet/aggregate/aggregateDependencies.ts

@@ -4,6 +4,8 @@ import { sanitizeDependencies } from "../utils/sanitizeDependencies.js";
 import { DappGetDnps } from "../types.js";
 import { DappGetFetcher } from "../fetch/index.js";
 import { DappnodeInstaller } from "../../dappnodeInstaller.js";
+import { logs } from "@dappnode/logger";
+import { Dependencies } from "@dappnode/types";
 
 /**
  * The goal of this function is to recursively aggregate all dependencies
@@ -27,13 +29,15 @@ export default async function aggregateDependencies({
   name,
   versionRange,
   dnps,
+  shouldThrow = false,
   recursiveCount,
   dappGetFetcher
 }: {
   dappnodeInstaller: DappnodeInstaller;
   name: string;
   versionRange: string;
   dnps: DappGetDnps;
+  shouldThrow?: boolean;
   recursiveCount?: number;
   dappGetFetcher: DappGetFetcher;
 }): Promise<void> {
@@ -55,29 +59,43 @@ export default async function aggregateDependencies({
       else setVersion(dnps, name, version, {});
       // 2. Get dependencies of this specific version
       //    dependencies = { dnp-name-1: "semverRange", dnp-name-2: "/ipfs/Qmf53..."}
-      const dependencies = await dappGetFetcher
-        .dependencies(dappnodeInstaller, name, version)
-        .then(sanitizeDependencies)
-        .catch((e: Error) => {
-          e.message += `Error fetching ${name}@${version}`;
-          throw e;
-        });
+      let dependencies: Dependencies;
+      try {
+        dependencies = await dappGetFetcher.dependencies(dappnodeInstaller, name, version).then(sanitizeDependencies);
+      } catch (e) {
+        if (shouldThrow) throw e;
+        // Remove this version if dependencies cannot be fetched
+        if (dnps[name] && dnps[name].versions) {
+          logs.debug(`[aggregateDependencies] Removing version ${name}@${version} due to fetch error: ${e?.message}`);
+          delete dnps[name].versions[version];
+        }
+        return;
+      }
 
-      // 3. Store dependencies
+      // 3. Store the dependency if it was fetched correctly
       setVersion(dnps, name, version, dependencies);
       // 4. Fetch sub-dependencies recursively
-      await Promise.all(
-        Object.keys(dependencies).map(async (dependencyName) => {
-          await aggregateDependencies({
-            dappnodeInstaller,
-            name: dependencyName,
-            versionRange: dependencies[dependencyName],
-            dnps,
-            recursiveCount,
-            dappGetFetcher
-          });
-        })
-      );
+      try {
+        await Promise.all(
+          Object.keys(dependencies).map(async (dependencyName) => {
+            await aggregateDependencies({
+              dappnodeInstaller,
+              name: dependencyName,
+              versionRange: dependencies[dependencyName],
+              dnps,
+              shouldThrow: true,
+              recursiveCount,
+              dappGetFetcher
+            });
+          })
+        );
+      } catch (e) {
+        // If any sub-dependency failed, remove this version
+        if (dnps[name] && dnps[name].versions) {
+          logs.debug(`[aggregateDependencies] Removing version ${name}@${version} due to sub-dependency failure`);
+          delete dnps[name].versions[version];
+        }
+      }
     })
   );
 }

packages/installer/src/dappGet/aggregate/cleanupDnp.ts

@@ -0,0 +1,42 @@
+import { DappGetDnps } from "../types.js";
+import { logs } from "@dappnode/logger";
+
+/**
+ * Cleans up the DNPs dependency graph by:
+ * 1. Removing DNPs with no versions left
+ * 2. Removing versions with missing dependencies
+ * Repeats until the graph is stable or a safety limit is reached. 
+ * Removing a version or a DNP may cause other DNPs or versions to become invalid, so we need to repeat the process.
+ * If we have not changed in an iteration, the graph is stable and we can stop.
+ */
+export function cleanupDnps(dnps: DappGetDnps) {
+  let changed = true;
+  let safety = 0;
+  const MAX_ITER = 1000;
+  while (changed) {
+    if (++safety > MAX_ITER) {
+      throw new Error("cleanupDnps: Exceeded max iterations, possible infinite loop");
+    }
+    changed = false;
+    // Remove DNPs with no versions
+    for (const dnpName of Object.keys(dnps)) {
+      if (Object.keys(dnps[dnpName].versions).length === 0) {
+        logs.debug(`[cleanupDnps] Removing DNP with no versions: ${dnpName}`);
+        delete dnps[dnpName];
+        changed = true;
+      }
+    }
+    // Remove versions with missing dependencies
+    for (const dnpName of Object.keys(dnps)) {
+      for (const version of Object.keys(dnps[dnpName].versions)) {
+        const deps = dnps[dnpName].versions[version];
+        const missingDeps = Object.keys(deps).filter(dep => !dnps[dep]);
+        if (missingDeps.length > 0) {
+          logs.debug(`[cleanupDnps] Removing version with missing dependencies: ${dnpName}@${version}, missing deps: ${missingDeps.join(", ")}`);
+          delete dnps[dnpName].versions[version];
+          changed = true;
+        }
+      }
+    }
+  }
+}

packages/installer/src/dappGet/aggregate/index.ts

@@ -3,6 +3,7 @@ import { params } from "@dappnode/params";
 // Internal
 import { safeSemver } from "../utils/safeSemver.js";
 import aggregateDependencies from "./aggregateDependencies.js";
+import { cleanupDnps } from "./cleanupDnp.js";
 import getRelevantInstalledDnps from "./getRelevantInstalledDnps.js";
 import { DappGetDnps } from "../types.js";
 import { logs } from "@dappnode/logger";
@@ -81,6 +82,14 @@ export default async function aggregate({
     dappGetFetcher // #### Injected dependency
   });
 
+  // If no resolvable packages or dependencies were found, throw an error
+  if (Object.keys(dnps).length === 0) {
+      throw new Error(`A dependency of the requested package "${req.name}" (version range: "${req.ver}") could not be resolved`);
+  }
+
+  // Clean up the dnps graph only once after all aggregation is done
+  cleanupDnps(dnps);
+
   const relevantInstalledDnps = getRelevantInstalledDnps({
     // requestedDnps = ["A", "B", "C"]
     requestedDnps: Object.keys(dnps),