fix: implement retry and caching for CLI downloads

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## UNRELEASED - CLI
 
 - fix: Make `modus --version` just print modus CLI's version [#563](https://github.com/hypermodeinc/modus/pull/563)
+- fix: implement retry and caching for CLI downloads [#571](https://github.com/hypermodeinc/modus/pull/571)
 
 ## UNRELEASED - Runtime
 

cli/package.json

@@ -34,6 +34,7 @@
     "chalk": "^5.3.0",
     "chokidar": "^4.0.1",
     "gradient-string": "^3.0.0",
+    "ky": "^1.7.2",
     "open": "^10.1.0",
     "ora": "^8.1.1",
     "picomatch": "^4.0.2",

cli/src/commands/new/index.ts

@@ -230,7 +230,8 @@ export default class NewCommand extends BaseCommand {
 
     // Verify and/or install the Modus SDK
     let installedSdkVersion = await vi.getLatestInstalledSdkVersion(sdk, prerelease);
-    if (await isOnline()) {
+    const isClientOnline = await isOnline();
+    if (isClientOnline) {
       let latestVersion: string | undefined;
       await withSpinner(chalk.dim(`Checking to see if you have the latest version of the ${sdkText}.`), async () => {
         latestVersion = await vi.getLatestSdkVersion(sdk, prerelease);
@@ -267,8 +268,13 @@ export default class NewCommand extends BaseCommand {
     }
 
     if (!installedSdkVersion) {
-      this.logError(`Could not find an installed ${sdkText}.`);
-      this.exit(1);
+      if (isClientOnline) {
+        this.logError(`Could not find an installed ${sdkText}.`);
+        this.exit(1);
+      } else {
+        this.logError(`Could not find a locally installed ${sdkText}. Please connect to the internet and try again.`);
+        this.exit(1);
+      }
     }
 
     const sdkVersion = installedSdkVersion;

cli/src/util/http.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2024 Hypermode Inc.
+ * Licensed under the terms of the Apache License, Version 2.0
+ * See the LICENSE file that accompanied this code for further details.
+ *
+ * SPDX-FileCopyrightText: 2024 Hypermode Inc. <[email protected]>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import ky from "ky";
+
+// All outbound HTTP requests in the CLI should be made through functions in this file,
+// to ensure consistent retry and timeout behavior.
+
+export async function get<T>(url: string, timeout: number | false = 10000) {
+  return await ky.get<T>(url, {
+    retry: 10, // retry 10 times, up to the default 10s timeout (unless overridden)
+    timeout,
+  });
+}

cli/src/util/index.ts

@@ -14,7 +14,9 @@ import path from "node:path";
 import { Readable } from "node:stream";
 import { finished } from "node:stream/promises";
 import { createWriteStream } from "node:fs";
+import * as http from "./http.js";
 import * as fs from "./fs.js";
+import * as vi from "./versioninfo.js";
 
 export async function withSpinner<T>(text: string, fn: (spinner: Ora) => Promise<T>): Promise<T> {
   // NOTE: Ora comes with "oraPromise", but it doesn't clear the original text on completion.
@@ -32,7 +34,7 @@ export async function withSpinner<T>(text: string, fn: (spinner: Ora) => Promise
 }
 
 export async function downloadFile(url: string, dest: string): Promise<boolean> {
-  const res = await fetch(url);
+  const res = await http.get(url, false);
   if (!res.ok) {
     console.log(chalk.red(" ERROR ") + chalk.dim(": Could not download file."));
     console.log(chalk.dim("   url : " + url));
@@ -56,16 +58,15 @@ let online: boolean | undefined;
 export async function isOnline(): Promise<boolean> {
   // Cache this, as we only need to check once per use of any CLI command that requires it.
   if (online !== undefined) return online;
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), 1000);
+
   try {
-    const response = await fetch(`https://releases.hypermode.com/modus-latest.json`, { signal: controller.signal });
-    online = response.ok;
+    // we don't need the result here, just checking if the request is successful
+    await vi.fetchModusLatest();
+    online = true;
   } catch {
     online = false;
-  } finally {
-    clearTimeout(timeout);
   }
+
   return online;
 }
 

cli/src/util/versioninfo.ts

@@ -9,6 +9,7 @@
 
 import semver from "semver";
 import path from "node:path";
+import * as http from "./http.js";
 import * as fs from "./fs.js";
 import * as globals from "../custom/globals.js";
 
@@ -27,44 +28,83 @@ export function isPrerelease(version: string): boolean {
   return !!semver.prerelease(version);
 }
 
-export async function fetchFromModusLatest(): Promise<{ [key: string]: string }> {
-  const response = await fetch(`https://releases.hypermode.com/modus-latest.json`, {});
+const versionData: {
+  latest?: { [key: string]: string };
+  preview?: { [key: string]: string };
+  all?: { [key: string]: string[] };
+  allPreview?: { [key: string]: string[] };
+} = {};
+
+export async function fetchModusLatest() {
+  if (versionData.latest) return versionData.latest;
+
+  const response = await http.get(`https://releases.hypermode.com/modus-latest.json`);
   if (!response.ok) {
-    throw new Error(`Error fetching latest SDK version: ${response.statusText}`);
+    throw new Error(`Error fetching latest SDK versions: ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as { [key: string]: string };
+  if (!data) {
+    throw new Error(`Error fetching latest SDK versions: response was empty`);
   }
 
-  return await response.json();
+  versionData.latest = data;
+  return data;
 }
 
-export async function fetchFromModusPreview(): Promise<{ [key: string]: string }> {
-  const response = await fetch(`https://releases.hypermode.com/modus-preview.json`, {});
+export async function fetchModusPreview() {
+  if (versionData.preview) return versionData.preview;
+
+  const response = await http.get(`https://releases.hypermode.com/modus-preview.json`);
   if (!response.ok) {
-    throw new Error(`Error fetching latest SDK version: ${response.statusText}`);
+    throw new Error(`Error fetching latest preview SDK versions: ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as { [key: string]: string };
+  if (!data) {
+    throw new Error(`Error fetching latest preview SDK versions: response was empty`);
   }
 
-  return await response.json();
+  versionData.preview = data;
+  return data;
 }
 
-export async function fetchFromModusAll(): Promise<{ [key: string]: string[] }> {
-  const response = await fetch(`https://releases.hypermode.com/modus-all.json`, {});
+export async function fetchModusAll() {
+  if (versionData.all) return versionData.all;
+
+  const response = await http.get(`https://releases.hypermode.com/modus-all.json`);
   if (!response.ok) {
     throw new Error(`Error fetching all SDK versions: ${response.statusText}`);
   }
 
-  return await response.json();
+  const data = (await response.json()) as { [key: string]: string[] };
+  if (!data) {
+    throw new Error(`Error fetching all SDK versions: response was empty`);
+  }
+
+  versionData.all = data;
+  return data;
 }
 
-export async function fetchFromModusPreviewAll(): Promise<{ [key: string]: string[] }> {
-  const response = await fetch(`https://releases.hypermode.com/modus-preview-all.json`, {});
+export async function fetchModusPreviewAll() {
+  if (versionData.allPreview) return versionData.allPreview;
+
+  const response = await http.get(`https://releases.hypermode.com/modus-preview-all.json`);
   if (!response.ok) {
-    throw new Error(`Error fetching all SDK versions: ${response.statusText}`);
+    throw new Error(`Error fetching all preview SDK versions: ${response.statusText}`);
+  }
+
+  const data = (await response.json()) as { [key: string]: string[] };
+  if (!data) {
+    throw new Error(`Error fetching all preview SDK versions: response was empty`);
   }
 
-  return await response.json();
+  versionData.allPreview = data;
+  return data;
 }
 
 export async function fetchItemVersionsFromModusAll(item: string): Promise<string[]> {
-  const data = await fetchFromModusAll();
+  const data = await fetchModusAll();
 
   if (item.endsWith("/")) {
     item = item.slice(0, -1);
@@ -79,7 +119,7 @@ export async function fetchItemVersionsFromModusAll(item: string): Promise<strin
 }
 
 export async function fetchItemVersionsFromModusPreviewAll(item: string): Promise<string[]> {
-  const data = await fetchFromModusPreviewAll();
+  const data = await fetchModusPreviewAll();
 
   if (item.endsWith("/")) {
     item = item.slice(0, -1);
@@ -94,19 +134,19 @@ export async function fetchItemVersionsFromModusPreviewAll(item: string): Promis
 }
 
 export async function getLatestSdkVersion(sdk: globals.SDK, includePrerelease: boolean): Promise<string | undefined> {
-  const data = includePrerelease ? await fetchFromModusPreview() : await fetchFromModusLatest();
+  const data = includePrerelease ? await fetchModusPreview() : await fetchModusLatest();
   const version = data["sdk/" + sdk.toLowerCase()];
   return version ? version : undefined;
 }
 
 export async function getLatestRuntimeVersion(includePrerelease: boolean): Promise<string | undefined> {
-  const data = includePrerelease ? await fetchFromModusPreview() : await fetchFromModusLatest();
+  const data = includePrerelease ? await fetchModusPreview() : await fetchModusLatest();
   const version = data["runtime"];
   return version ? version : undefined;
 }
 
 export async function getLatestCliVersion(includePrerelease: boolean): Promise<string | undefined> {
-  const data = includePrerelease ? await fetchFromModusPreview() : await fetchFromModusLatest();
+  const data = includePrerelease ? await fetchModusPreview() : await fetchModusLatest();
   const version = data["cli"];
   return version ? version : undefined;
 }