Merge remote-tracking branch 'origin/master' into llm-tweak-abuse

Author: haraldschilly

.github/workflows/make-and-test.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Install python3 requests
         run: sudo apt-get install python3-requests
       - name: Check doc links
-        run: cd src/scripts && python3 check_doc_urls.py
+        run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py
 
   build:
     runs-on: ubuntu-latest

src/packages/database/postgres/schema/table.ts

@@ -39,6 +39,9 @@ export async function createTable(
     if (info.unique) {
       s += " UNIQUE";
     }
+    if (info.not_null) {
+      s += " NOT NULL";
+    }
     if (info.pg_check) {
       s += " " + info.pg_check;
     }

src/packages/database/settings/customize.ts

@@ -58,6 +58,7 @@ export interface Customize {
   neuralSearchEnabled?: boolean;
   jupyterApiEnabled?: boolean;
   computeServersEnabled?: boolean;
+  cloudFilesystemsEnabled?: boolean;
   githubProjectId?: string;
   support?: string;
 }
@@ -167,6 +168,7 @@ export default async function getCustomize(): Promise<Customize> {
     jupyterApiEnabled: settings.jupyter_api_enabled,
 
     computeServersEnabled: settings.compute_servers_enabled,
+    cloudFilesystemsEnabled: settings.cloud_filesystems_enabled,
 
     // GitHub proxy project
     githubProjectId: settings.github_project_id,

src/packages/frontend/account/account-page.tsx

@@ -29,6 +29,8 @@ import { UpgradesPage } from "./upgrades/upgrades-page";
 import PurchasesPage from "@cocalc/frontend/purchases/purchases-page";
 import SubscriptionsPage from "@cocalc/frontend/purchases/subscriptions-page";
 import StatementsPage from "@cocalc/frontend/purchases/statements-page";
+import { cloudFilesystemsEnabled } from "@cocalc/frontend/compute";
+import CloudFilesystems from "@cocalc/frontend/compute/cloud-filesystem/cloud-filesystems";
 
 export const AccountPage: React.FC = () => {
   const active_page = useTypedRedux("account", "active_page");
@@ -218,6 +220,17 @@ export const AccountPage: React.FC = () => {
         children: active_page === "upgrades" && <UpgradesPage />,
       });
     }
+    if (cloudFilesystemsEnabled()) {
+      items.push({
+        key: "cloud-filesystems",
+        label: (
+          <>
+            <Icon name="disk-round" /> Cloud Filesystems
+          </>
+        ),
+        children: <CloudFilesystems />,
+      });
+    }
 
     return items;
   }

src/packages/frontend/client/api.ts

@@ -32,12 +32,13 @@ async function callApi(endpoint: string, args?: object) {
     },
     ...(args != null ? { body: JSON.stringify(args) } : undefined),
   });
-  // const respClone = resp.clone();
+  const respClone = resp.clone();
   let json: any = null;
   try {
     json = await resp.json();
   } catch (e) {
-    // console.log(await respClone.text());
+    console.log(e);
+    console.log(await respClone.text());
     throw Error("API server is down -- try again later");
   }
   if (json == null) {

src/packages/frontend/client/project.ts

@@ -20,6 +20,7 @@ import * as message from "@cocalc/util/message";
 import { DirectoryListingEntry } from "@cocalc/util/types";
 import { connection_to_project } from "../project/websocket/connect";
 import { API } from "../project/websocket/api";
+import httpApi from "./api";
 import { redux } from "../app-framework";
 import { WebappClient } from "./client";
 import { allow_project_to_run } from "../project/client-side-throttle";
@@ -36,31 +37,7 @@ import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
 import { ipywidgetsGetBufferUrl } from "@cocalc/frontend/jupyter/server-urls";
 import type { ApiKey } from "@cocalc/util/db-schema/api-keys";
 import computeServers from "@cocalc/frontend/compute/manager";
-
-export interface ExecOpts {
-  project_id: string;
-  compute_server_id?: number; // if true, run on the compute server (if available)
-  filesystem?: boolean; // run in fileserver container on compute server; otherwise, runs on main compute container.
-  path?: string;
-  command: string;
-  args?: string[];
-  timeout?: number;
-  max_output?: number;
-  bash?: boolean;
-  aggregate?: string | number | { value: string | number };
-  err_on_exit?: boolean;
-  env?: { [key: string]: string }; // custom environment variables.
-  cb?: Function; // if given use a callback interface *instead* of async.
-}
-export const ExecOpts = null; // webpack + TS es2020 modules need this
-
-export interface ExecOutput {
-  stdout: string;
-  stderr: string;
-  exit_code: number;
-  time: number; // time in ms, from user point of view.
-}
-export const ExecOutput = null; // webpack + TS es2020 modules need this
+import type { ExecOpts, ExecOutput } from "@cocalc/util/db-schema/projects";
 
 export class ProjectClient {
   private client: WebappClient;
@@ -208,7 +185,7 @@ export class ProjectClient {
     time" (which is stored in the db), which they client will know.  This is used, e.g.,
     for operations like "run rst2html on this file whenever it is saved."
     */
-  public async exec(opts: ExecOpts): Promise<ExecOutput> {
+  public async exec(opts: ExecOpts & { post?: boolean }): Promise<ExecOutput> {
     opts = defaults(opts, {
       project_id: required,
       compute_server_id: undefined,
@@ -222,6 +199,7 @@ export class ProjectClient {
       aggregate: undefined,
       err_on_exit: true,
       env: undefined,
+      post: false, // if true, uses the POST api through nextjs instead of the websocket api.
       cb: undefined, // if given use a callback interface instead of async
     });
 
@@ -239,11 +217,19 @@ export class ProjectClient {
       };
     }
 
-    try {
-      const ws = await this.websocket(opts.project_id);
-      const exec_opts = copy_without(opts, ["project_id"]);
+    const { post } = opts;
+    delete opts.post;
 
-      const msg = await ws.api.exec(exec_opts);
+    try {
+      let msg;
+      if (post) {
+        // use post API
+        msg = await httpApi("exec", opts);
+      } else {
+        const ws = await this.websocket(opts.project_id);
+        const exec_opts = copy_without(opts, ["project_id"]);
+        msg = await ws.api.exec(exec_opts);
+      }
       if (msg.status && msg.status == "error") {
         throw new Error(msg.error);
       }

src/packages/frontend/components/plotly.tsx

@@ -0,0 +1,16 @@
+import { useRef } from "react";
+import ReactPlotly from "react-plotly.js";
+import useResizeObserver from "use-resize-observer";
+
+export default function Plot(props) {
+  const divRef = useRef<HTMLDivElement>(null);
+  const resize = useResizeObserver({ ref: divRef });
+  return (
+    <div ref={divRef} style={props.style}>
+      <ReactPlotly
+        {...props}
+        layout={{ ...props.layout, width: resize.width }}
+      />
+    </div>
+  );
+}

src/packages/frontend/compute/api.ts

@@ -1,4 +1,5 @@
 import api from "@cocalc/frontend/client/api";
+("");
 import type {
   Action,
   ComputeServerTemplate,
@@ -123,6 +124,23 @@ export const getGoogleCloudPriceData = reuseInFlight(
   },
 );
 
+import { useState, useEffect } from "react";
+export function useGoogleCloudPriceData() {
+  const [priceData, setPriceData] = useState<null | GoogleCloudData>(null);
+  const [error, setError] = useState<string>("");
+  useEffect(() => {
+    (async () => {
+      try {
+        setError("");
+        setPriceData(await getGoogleCloudPriceData());
+      } catch (err) {
+        setError(`${err}`);
+      }
+    })();
+  }, []);
+  return [priceData, error];
+}
+
 // Cache for 5 minutes -- cache less since this includes realtime
 // information about GPU availability.
 let hyperstackPriceData: HyperstackPriceData | null = null;
@@ -168,9 +186,17 @@ export async function getLog(opts: { id }) {
   return await api("compute/get-log", opts);
 }
 
-export const getTitle = reuseInFlight(async (opts: { id: number }) => {
-  return await api("compute/get-server-title", opts);
-});
+export const getTitle = reuseInFlight(
+  async (opts: {
+    id: number;
+  }): Promise<{
+    title: string;
+    color: string;
+    project_specific_id: number;
+  }> => {
+    return await api("compute/get-server-title", opts);
+  },
+);
 
 // Setting a detailed state component for a compute server
 export async function setDetailedState(opts: {

src/packages/frontend/compute/check-in.ts

@@ -0,0 +1,47 @@
+import { CHECK_IN_PATH } from "@cocalc/util/db-schema/compute-servers";
+import { webapp_client } from "@cocalc/frontend/webapp-client";
+import { redux } from "@cocalc/frontend/app-framework";
+
+// cause the given compute server to check in, or all running compute
+// servers in the project if compute_server_id isn't specified.
+export async function checkIn({
+  project_id,
+  compute_server_id,
+}: {
+  project_id: string;
+  compute_server_id: number;
+}) {
+  if (compute_server_id != null) {
+    await webapp_client.project_client.exec({
+      command: "touch",
+      args: [CHECK_IN_PATH],
+      project_id,
+      compute_server_id,
+    });
+    return;
+  }
+}
+
+// launches in parallel check in on all running projects; doesn't
+// wait for response.
+export function checkInAll(project_id) {
+  const computeServers = redux
+    .getProjectStore(project_id)
+    .get("compute_servers");
+  if (computeServers == null) {
+    return;
+  }
+  const ids = computeServers
+    .filter((x) => x.get("state") == "running")
+    .map((x) => x.get("id"))
+    .keySeq();
+  for (const id of ids) {
+    (async () => {
+      try {
+        await checkIn({ project_id, compute_server_id: id });
+      } catch (err) {
+        console.warn(`checkIn issue with compute server ${id}`, err);
+      }
+    })();
+  }
+}

src/packages/frontend/compute/cloud-filesystem/api.ts

@@ -0,0 +1,86 @@
+import api from "@cocalc/frontend/client/api";
+import LRU from "lru-cache";
+import type {
+  CreateCloudFilesystem,
+  CloudFilesystem,
+  EditCloudFilesystem,
+} from "@cocalc/util/db-schema/cloud-filesystems";
+import {
+  CHANGE_MOUNTED,
+  CHANGE_UNMOUNTED,
+} from "@cocalc/util/db-schema/cloud-filesystems";
+
+export async function createCloudFilesystem(
+  opts: CreateCloudFilesystem,
+): Promise<number> {
+  return await api("compute/cloud-filesystem/create", opts);
+}
+
+export async function deleteCloudFilesystem({
+  id,
+  lock,
+}: {
+  id: number;
+  lock: string;
+}): Promise<void> {
+  await api("compute/cloud-filesystem/delete", { id, lock });
+}
+
+export async function editCloudFilesystem(
+  opts: EditCloudFilesystem,
+): Promise<void> {
+  for (const field in opts) {
+    if (field == "id") {
+      continue;
+    }
+    if (!CHANGE_MOUNTED.has(field) && !CHANGE_UNMOUNTED.has(field)) {
+      throw Error(`invalid field '${field}' for edit`);
+    }
+  }
+  return await api("compute/cloud-filesystem/edit", opts);
+}
+
+const cloudFilesystemCache = new LRU<string, CloudFilesystem[]>({
+  max: 100,
+  ttl: 1000 * 30,
+});
+
+export async function getCloudFilesystems(
+  // give no options to get all filesystems that YOU own across all your projects
+  opts: {
+    // id = specific on
+    id?: number;
+    // project_id = all in a given project (owned by anybody)
+    project_id?: string;
+    // if true, use cache and potentially return stale data (good for repeated calls in a list, etc.)
+    cache?: boolean;
+  } = {},
+): Promise<CloudFilesystem[]> {
+  const key = `${opts.id}${opts.project_id}`;
+  if (opts.cache && cloudFilesystemCache.has(key)) {
+    return cloudFilesystemCache.get(key)!;
+  }
+  const filesystems = await api("compute/cloud-filesystem/get", {
+    id: opts.id,
+    project_id: opts.project_id,
+  });
+  // always save in cache
+  cloudFilesystemCache.set(key, filesystems);
+  return filesystems;
+}
+
+export async function getMetrics({
+  id,
+  limit,
+  offset,
+}: {
+  id: number;
+  limit?: number;
+  offset?: number;
+}) {
+  return await api("compute/cloud-filesystem/get-metrics", {
+    cloud_filesystem_id: id,
+    limit,
+    offset,
+  });
+}