feat: support for server-side log aggregation

starpit · starpit · commit 5acf2d29d4ec · 2022-08-10T11:51:12.000-04:00
Also:
- general fixes for dockerized execution
- codeflare dashboard -a updates
- dashboard -a does not accept -p
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1 @@
+store/**/*.md
diff --git a/bin/codeflare b/bin/codeflare
@@ -19,6 +19,9 @@ SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 if [ -n "$NODE" ] && [ -n "$HEADLESS" ]; then
     # no-op; use built-in node
     ______USING_BUILTIN=true # not needed, just making bash happy
+elif [ -n "$CODEFLARE_HEADLESS_HOME" ]; then
+    NODE=node
+    HEADLESS="$CODEFLARE_HEADLESS_HOME"
 elif [ -f ${SCRIPTDIR}/../node_modules/electron/dist/Electron.app/Contents/MacOS/Electron ]; then
     # development mode on MacOS
     NODE=${SCRIPTDIR}/../node_modules/electron/dist/Electron.app/Contents/MacOS/Electron
@@ -100,6 +103,9 @@ if [ -z "$GUIDEBOOK_STORE" ]; then
     if [  -d "$HEADLESS"/../../store ]; then
         # development builds
         export GUIDEBOOK_STORE="$HEADLESS"/../../store
+    elif [  -d "$HEADLESS"/store ]; then
+        # docker builds
+        export GUIDEBOOK_STORE="$HEADLESS"/store
     else
         # otherwise, we can't find a local mirror, so pull directly from
         # git (network transfers!)
@@ -134,7 +140,7 @@ if [ "$do_cli" = "1" ]; then
 fi
 
 # Linux may not have the prereqs needed to run Electron
-if [ ! -f ~/.codeflare ] && [ $(uname) = Linux ]; then
+if [ "$NODE" != "node" ] && [ ! -f ~/.codeflare ] && [ $(uname) = Linux ]; then
     if [ "$(grep '^ID_LIKE' /etc/os-release)" = "ID_LIKE=debian" ]; then
         if [ $(apt -qq list sudo curl libx11-6 libglib2.0-0 libx11-xcb1 libxcb-dri3-0 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxi6 libxtst6 libnss3 libatk1.0-0 libdrm2 libgbm1 libatk-bridge2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libasound2 | grep installed 2> /dev/null | wc -l) != 20 ]; then
             # it's also possible we didn't see these in the list, because we have an out-of-date package list
@@ -153,6 +159,9 @@ if [ ! -f ~/.codeflare ] && [ $(uname) = Linux ]; then
     else
         touch ~/.codeflare
     fi
+
+    # our docker build uses this to do the apt initialization only
+    if [ -n "$CODEFLARE_INIT_FOR_DOCKER" ]; then exit 0; fi
 fi
 
 # helps with ctrl+c when running in a docker container
diff --git a/deploy/cli/Dockerfile b/deploy/cli/Dockerfile
@@ -1,31 +1,33 @@
-FROM ubuntu:latest
+FROM node:lts-slim
 
 ENV DEBIAN_FRONTEND=noninteractive
-ENV PATH=/usr/local/CodeFlare/bin:$PATH
+ENV KUI_HEADLESS=true
+ENV KUI_HEADLESS_WEBPACK=true
+ENV CODEFLARE_HEADLESS_HOME=/usr/local/CodeFlare
 
 # TODO hard-coded arch
-ADD dist/electron/CodeFlare-linux-x64 /usr/local/CodeFlare
+ADD dist/headless /usr/local/CodeFlare
+ADD store /usr/local/CodeFlare/store
+ADD bin/codeflare /usr/local/bin/codeflare
 
-# Note, the `codeflare ...` commands are not strictly necessary. We do
-# them here only to save some time installing these prereqs on every
-# run.
-RUN apt update && apt -y install python3 python3-pip \
+RUN apt update && apt -y install python3 python3-pip sudo curl \
         && apt -y clean && rm -rf /var/lib/apt/lists/* \
-        && codeflare util/jq \
-        && codeflare util/websocat \
-        && codeflare s3/install/cli \
-        && codeflare ml/ray/install/cli \
-        && codeflare kubernetes/kubectl \
-        && codeflare kubernetes/helm3 \
+        && (cd /usr/bin && ln -s pip3 pip) \
         && adduser --disabled-password --gecos '' codeflare && adduser codeflare sudo && adduser root sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
-        && touch /home/codeflare/.codeflare 
+        && mkdir -p /root/.local/share/madwizard-nodejs/profiles \
+        && codeflare -n util/jq \
+        && codeflare -n util/websocat \
+        && codeflare -n s3/install/cli \
+        && codeflare -n ml/ray/install/cli \
+        && codeflare -n kubernetes/kubectl \
+        && codeflare -n kubernetes/helm3
 
-ENTRYPOINT ["codeflare"]
-WORKDIR /home/codeflare
 USER codeflare
+WORKDIR /home/codeflare
 
-
-
+# Note, the `codeflare ...` commands are not strictly necessary. We do
+# them here only to save some time installing these prereqs on every
+# run.
 
 # in case we want to use the published production versions at some point...
 #RUN export FILE=CodeFlare-linux-$([ "$(uname -m)" = "x86_64" ] && echo x64 || echo arm64) \
diff --git a/deploy/cli/codeflare.yaml b/deploy/cli/codeflare.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: codeflare
+  labels:
+    app: codeflare
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: codeflare
+  template:
+    metadata:
+      labels:
+        app: codeflare
+    spec:
+      containers:
+      - name: codeflare
+        image: ghcr.io/project-codeflare/codeflare-cli
diff --git a/deploy/log-aggregator/Dockerfile b/deploy/log-aggregator/Dockerfile
@@ -0,0 +1,39 @@
+FROM node:lts-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV KUI_HEADLESS=true
+ENV KUI_HEADLESS_WEBPACK=true
+ENV CODEFLARE_HEADLESS_HOME=/usr/local/CodeFlare
+
+# TODO hard-coded arch
+ADD dist/headless /usr/local/CodeFlare
+ADD store /usr/local/CodeFlare/store
+ADD bin/codeflare /usr/local/bin/codeflare
+ADD deploy/log-aggregator/wait-for.sh /usr/local/bin/wait-for
+ADD deploy/log-aggregator/wait-for-and-cat.sh /usr/local/bin/wait-for-and-cat
+ADD deploy/log-aggregator/wait-for-and-tailf.sh /usr/local/bin/wait-for-and-tailf
+
+# Note, the `codeflare ...` commands are not strictly necessary. We do
+# them here only to save some time installing these prereqs on every
+# run.
+RUN apt update && apt -y install sudo curl procps inotify-tools \
+        && apt -y clean && rm -rf /var/lib/apt/lists/* \
+        && adduser --disabled-password --gecos '' codeflare && adduser codeflare sudo && adduser root sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+        && mkdir -p /root/.local/share/madwizard-nodejs/profiles \
+        && codeflare -n util/jq \
+        && codeflare -n util/websocat \
+        && codeflare -n kubernetes/kubectl
+
+USER codeflare
+WORKDIR /home/codeflare
+
+CMD exec /bin/bash -c "trap : TERM INT; sleep infinity & wait"
+# ENTRYPOINT ["codeflare", "-y", "-p", "default", "ml/ray/aggregator/with-jobid"]
+
+
+# Notes
+# - KUBE_CONTEXT_ARG must be ""
+# - KUBE_NS_ARG can be "" for kubectl, but since we need it for RAY_ADDRESS, use the actual `-n ${KUBE_NS}` value
+# - RAY_ADDRESS must be of the form mycluster-ray-head.ray.svc.cluster.local:8265
+#                                   $RAY_KUBE_CLUSTER_NAME.$KUBE_NS.svc.cluster.local:8265
+#                                                           ^^^^^^^ must be the actual namespace value
diff --git a/deploy/log-aggregator/wait-for-and-cat.sh b/deploy/log-aggregator/wait-for-and-cat.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+F="$1"
+
+wait-for "$F" && cat "$F"
diff --git a/deploy/log-aggregator/wait-for-and-tailf.sh b/deploy/log-aggregator/wait-for-and-tailf.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+F="$1"
+
+wait-for "$F" && tail -f -n +1 "$F"
diff --git a/deploy/log-aggregator/wait-for.sh b/deploy/log-aggregator/wait-for.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+F="$1"
+
+# wait for the file to exist and be non-empty
+while [ ! -s "$F" ]; do
+    mkdir -p $(dirname $F)
+    inotifywait -qq -e create -e modify "$(dirname $F)"
+done
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -19,8 +19,10 @@
     "pty:nodejs": "npm run pty:rebuild node",
     "prepare": "cross-env husky install",
     "preprod": "npm run mirror",
-    "build:docker0": "docker build -t ghcr.io/project-codeflare/codeflare-cli .",
-    "build:docker": "NO_INSTALLER=true npm run build:electron:linux:amd64 && npm run build:docker0",
+    "build:docker:cli": "npm run build:docker1 && docker build -f deploy/cli/Dockerfile -t ghcr.io/project-codeflare/codeflare-cli .",
+    "build:docker:logs": "npm run build:docker1 && docker build -f deploy/log-aggregator/Dockerfile -t ghcr.io/project-codeflare/codeflare-log-aggregator .",
+    "build:docker1": "if [ ! -n \"$FAST\" ]; then npm run build:docker0; fi",
+    "build:docker0": "npm run mirror && CLIENT_HOME=$PWD webpack-cli --mode=${MODE-production} --config node_modules/@kui-shell/webpack/headless-webpack.config.js",
     "docker:debug": "ENTRYPOINT=bash ./bin/codeflare -d",
     "build:electron:mac:amd64": "npm run preprod && KUI_HEADLESS_WEBPACK=true KUI_LAUNCHER=$PWD/bin/codeflare PLATFORM=mac ARCH=x64 kui-build-electron",
     "build:electron:mac:arm64": "npm run preprod && KUI_HEADLESS_WEBPACK=true KUI_LAUNCHER=$PWD/bin/codeflare PLATFORM=mac ARCH=arm64 kui-build-electron",
diff --git a/plugins/plugin-codeflare/src/controller/attach.ts b/plugins/plugin-codeflare/src/controller/attach.ts
@@ -14,29 +14,87 @@
  * limitations under the License.
  */
 
-import { cli } from "madwizard/dist/fe/cli"
-import { MadWizardOptions, Profiles } from "madwizard"
-import { Arguments, encodeComponent } from "@kui-shell/core"
-
-export default async function attach(args: Arguments) {
-  const options: MadWizardOptions = {
-    profile: await Profiles.lastUsed(),
-    store: process.env.GUIDEBOOK_STORE,
-    clean: false,
-    interactive: args.parsedOptions.i || !args.parsedOptions.y,
+import { MadWizardOptions } from "madwizard"
+import { Arguments, ParsedOptions } from "@kui-shell/core"
+
+import { DashboardOptions } from "./dashboard"
+
+type Options = ParsedOptions &
+  DashboardOptions & {
+    /** verbose output from madwizard */
+    V?: boolean
   }
 
+/**
+ * Attach to the given `jobId` running in the given `profile`
+ *
+ * @return the path to the captured log directory and a cleanup
+ * handler to be invoked when you are done consuming the output
+ */
+export async function attach(profile: string, jobId: boolean | string, opts: MadWizardOptions = {}) {
+  const deployGuidebook = "ml/ray/aggregator/in-cluster/client-side/deploy"
+  const startGuidebook =
+    typeof jobId === "string"
+      ? "ml/ray/aggregator/in-cluster/client-side/start/with-jobId"
+      : "ml/ray/aggregator/in-cluster/client-side/start"
+
   // TODO: update madwizard to accept env in the options
   process.env.NO_WAIT = "true" // don't wait for job termination
   process.env.QUIET_CONSOLE = "true" // don't tee logs to the console
+  if (typeof jobId === "string") {
+    process.env.JOB_ID = jobId
+  }
+
+  const options: MadWizardOptions = Object.assign(
+    {
+      profile,
+      clean: false,
+      interactive: false,
+      verbose: true,
+      store: process.env.GUIDEBOOK_STORE,
+    },
+    opts
+  )
 
-  const resp = await cli(["codeflare", "guide", "ml/ray/aggregator"], undefined, options)
+  // *deploy* the aggregator pod, then *start* an aggregator instance
+  // for the given jobId
+  const { cli } = await import("madwizard/dist/fe/cli")
+  await cli(
+    ["codeflare", "guide", deployGuidebook],
+    undefined,
+    Object.assign({}, options, { name: "log-aggregator-deploy" })
+  )
+  const resp = await cli(
+    ["codeflare", "guide", startGuidebook],
+    undefined,
+    Object.assign({}, options, { name: "log-aggregator-start" })
+  )
 
+  // the logdir that we captured
   const logdir = resp && resp.env ? resp.env.LOGDIR_STAGE : undefined
 
-  if (logdir) {
-    process.env.LOGDIR = logdir
-    await args.REPL.qexec(`codeflare dashboardui -f ${encodeComponent(logdir)}`)
+  return {
+    logdir,
+    cleanExit: resp ? resp.cleanExit : undefined,
+  }
+}
+
+/**
+ * Kui command handler that wraps `attach`, and opens a dashboard
+ * window that visualizes the contents the produced log directory.
+ */
+export default async function attachCommand(args: Arguments<Options>) {
+  const [{ encodeComponent }, { Profiles }] = await Promise.all([import("@kui-shell/core"), import("madwizard")])
+
+  const jobId = args.parsedOptions.a
+  const profile = args.parsedOptions.p || (await Profiles.lastUsed())
+
+  const resp = await attach(profile, jobId, { verbose: args.parsedOptions.V })
+
+  if (resp.logdir) {
+    process.env.LOGDIR = resp.logdir
+    await args.REPL.qexec(`codeflare dashboardui -f ${encodeComponent(resp.logdir)}`)
+    // TODO utilize resp.cleanExit?
   } else {
     throw new Error("Could not attach, due to missing logging directory")
   }
diff --git a/plugins/plugin-codeflare/src/controller/dashboard.ts b/plugins/plugin-codeflare/src/controller/dashboard.ts
@@ -25,14 +25,17 @@ export interface FollowOptions {
   follow: boolean
 }
 
-type DashboardOptions = FollowOptions & {
-  a: boolean
-  attach: boolean
+export type DashboardOptions = FollowOptions & {
+  a: boolean | string
+  attach: boolean | string
+
+  p: string
+  profile: string
 }
 
 export const followFlags: CommandOptions["flags"] = {
-  boolean: ["f", "follow", "a", "attach"],
-  alias: { follow: ["f"], attach: ["a"] },
+  boolean: ["f", "follow", "i", "y", "V"],
+  alias: { follow: ["f"], attach: ["a"], profile: ["p"] },
 }
 
 function dashboardcli(args: Arguments<DashboardOptions>) {
@@ -76,7 +79,7 @@ export default function registerDashboardCommands(registrar: Registrar) {
     needsUI: true,
     outputOnly: true,
     flags,
-    title: (argv: string[]) => productName + " Dashboard: " + basename(argv[2]),
+    title: (argv: string[]) => productName + " Dashboard: " + basename(argv.filter((_) => !/^-/.test(_))[2]),
     width,
     height,
   })
diff --git a/plugins/plugin-codeflare/src/tray/menus/profiles/active-runs.ts b/plugins/plugin-codeflare/src/tray/menus/profiles/active-runs.ts
diff --git a/plugins/plugin-madwizard/package.json b/plugins/plugin-madwizard/package.json
diff --git a/plugins/plugin-madwizard/src/plugin.ts b/plugins/plugin-madwizard/src/plugin.ts

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +#!/usr/bin/env bash
++
 +F="$1"
++
 +wait-for "$F" && cat "$F"