server/conat: discover peers only via K8S API

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: haraldschilly

src/.claude/settings.local.json

@@ -0,0 +1,11 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pnpm tsc:*)",
+      "Bash(pnpm build:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)"
+    ],
+    "deny": []
+  }
+}

src/CLAUDE.md

@@ -0,0 +1,130 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+# CoCalc Source Repository
+
+* This is the source code of CoCalc in a Git repository
+* It is a complex JavaScript/TypeScript SaaS application
+* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
+* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
+
+## Code Style
+
+- Everything is written in TypeScript code
+- Indentation: 2-spaces
+- All .js and .ts files are formatted by the tool prettier
+- Add suitable types when you write code
+- Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style.
+- Some older code is JavaScript or CoffeeScript, which will be translated to TypeScript
+- Use ES modules (import/export) syntax, not CommonJS (require)
+- Organize the list of imports in such a way: installed npm packages are on top, newline, then are imports from @cocalc's code base. Sorted alphabetically.
+
+## Development Commands
+
+### Essential Commands
+- `pnpm build-dev` - Build all packages for development
+- `pnpm clean` - Clean all node_modules and dist directories
+- `pnpm database` - Start PostgreSQL database server
+- `pnpm hub` - Start the main hub server
+- `pnpm psql` - Connect to the PostgreSQL database
+- `pnpm test` - Run full test suite
+- `pnpm test-parallel` - Run tests in parallel across packages
+- `pnpm depcheck` - Check for dependency issues
+
+### Package-Specific Commands
+- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package
+- `cd packages/[package] && pnpm test` - Run tests for a specific package
+- `cd packages/[package] && pnpm build` - Build a specific package
+
+### Development Setup
+1. Start database: `pnpm database`
+2. Start hub: `pnpm hub`
+3. For TypeScript changes, run `pnpm tsc` in the relevant package directory
+
+## Architecture Overview
+
+### Package Structure
+CoCalc is organized as a monorepo with key packages:
+
+- **frontend** - React/TypeScript frontend application using Redux-style stores and actions
+- **backend** - Node.js backend services and utilities
+- **hub** - Main server orchestrating the entire system
+- **database** - PostgreSQL database layer with queries and schema
+- **util** - Shared utilities and types used across packages
+- **comm** - Communication layer including WebSocket types
+- **conat** - CoCalc's container/compute orchestration system
+- **sync** - Real-time synchronization system for collaborative editing
+- **project** - Project-level services and management
+- **static** - Static assets and build configuration
+- **next** - Next.js server components
+
+### Key Architectural Patterns
+
+#### Frontend Architecture
+- **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`)
+- **TypeScript React Components**: All frontend code is TypeScript with proper typing
+- **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
+- **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+
+#### Backend Architecture
+- **PostgreSQL Database**: Primary data store with sophisticated querying system
+- **WebSocket Messaging**: Real-time communication between frontend and backend
+- **Conat System**: Container orchestration for compute servers
+- **Event-Driven Architecture**: Extensive use of EventEmitter patterns
+- **Microservice-like Packages**: Each package handles specific functionality
+
+#### Communication Patterns
+- **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`)
+- **Database Queries**: Structured query system with typed interfaces
+- **Event Emitters**: Inter-service communication within backend
+- **REST-like APIs**: Some HTTP endpoints for specific operations
+
+### Key Technologies
+- **TypeScript**: Primary language for all new code
+- **React**: Frontend framework
+- **PostgreSQL**: Database
+- **Node.js**: Backend runtime
+- **WebSockets**: Real-time communication
+- **pnpm**: Package manager and workspace management
+- **Jest**: Testing framework
+- **SASS**: CSS preprocessing
+
+### Database Schema
+- Comprehensive schema in `packages/util/db-schema`
+- Query abstractions in `packages/database/postgres/`
+- Type-safe database operations with TypeScript interfaces
+
+### Testing
+- **Jest**: Primary testing framework
+- **ts-jest**: TypeScript support for Jest
+- **jsdom**: Browser environment simulation for frontend tests
+- Test files use `.test.ts` or `.spec.ts` extensions
+- Each package has its own jest.config.js
+
+### Import Patterns
+- Use absolute imports with `@cocalc/` prefix for cross-package imports
+- Example: `import { cmp } from "@cocalc/util/misc"`
+- Type imports: `import type { Foo } from "./bar"`
+- Destructure imports when possible
+
+### Development Workflow
+1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package)
+2. Database must be running before starting hub
+3. Hub coordinates all services and should be restarted after changes
+4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes
+
+# Workflow
+- Be sure to typecheck when you're done making a series of code changes
+- Prefer running single tests, and not the whole test suite, for performance
+
+## Git Workflow
+
+- Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
+- When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
+
+# important-instruction-reminders
+- Do what has been asked; nothing more, nothing less.
+- NEVER create files unless they're absolutely necessary for achieving your goal.
+- ALWAYS prefer editing an existing file to creating a new one.
+- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.

src/packages/server/conat/socketio/dns-scan-k8s-api.ts

@@ -1,36 +1,29 @@
 import { readFile } from "fs/promises";
 import * as https from "https";
 
+import type { PodInfos } from "./dns-scan";
+
 // Define the options interface for type safety
 interface ListPodsOptions {
   labelSelector?: string; // e.g. "app=foo,env=prod"
 }
 
 let NAMESPACE: string | null = null;
-let CA: Buffer | null = null;
+let CA: string | null = null;
+
+async function readK8sFile(filename: string): Promise<string> {
+  const basePath = "/var/run/secrets/kubernetes.io/serviceaccount";
+  return (await readFile(`${basePath}/${filename}`, "utf8")).trim();
+}
 
 async function listPods(options: ListPodsOptions = {}): Promise<any> {
   let token: string;
   try {
-    NAMESPACE =
-      NAMESPACE ??
-      (
-        await readFile(
-          "/var/run/secrets/kubernetes.io/serviceaccount/namespace",
-          "utf8",
-        )
-      ).trim();
-    CA =
-      CA ??
-      (await readFile("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"));
+    NAMESPACE ??= await readK8sFile("namespace");
+    CA ??= await readK8sFile("ca.crt");
 
     // Read service account details, token could be rotated, so read every time
-    token = (
-      await readFile(
-        "/var/run/secrets/kubernetes.io/serviceaccount/token",
-        "utf8",
-      )
-    ).trim();
+    token = await readK8sFile("token");
   } catch (err) {
     throw new Error(`Failed to read service account files: ${err}`);
   }
@@ -88,11 +81,9 @@ async function listPods(options: ListPodsOptions = {}): Promise<any> {
   });
 }
 
-export async function getAddressesFromK8sApi(): Promise<
-  { name: string; podIP: string }[]
-> {
+export async function getAddressesFromK8sApi(): Promise<PodInfos> {
   const res = await listPods({ labelSelector: "run=hub-conat-router" });
-  const ret: { name: string; podIP: string }[] = [];
+  const ret: PodInfos = [];
   for (const pod of res.items) {
     const name = pod.metadata?.name;
     const podIP = pod.status?.podIP;

src/packages/server/conat/socketio/dns-scan.ts

@@ -8,28 +8,14 @@ import { delay } from "awaiting";
 import { lookup } from "dns/promises";
 import { hostname } from "node:os";
 
-import { executeCode } from "@cocalc/backend/execute-code";
 import { getLogger } from "@cocalc/backend/logger";
 import port from "@cocalc/backend/port";
 import type { ConatServer } from "@cocalc/conat/core/server";
-import { split, unreachable } from "@cocalc/util/misc";
 import { getAddressesFromK8sApi } from "./dns-scan-k8s-api";
 
 export const SCAN_INTERVAL = 15_000;
 
-type PeerDiscovery = "KUBECTL" | "API";
-
-function isPeerDiscovery(x: string): x is PeerDiscovery {
-  return x === "KUBECTL" || x === "API";
-}
-
-const PEER_DISCOVERY: PeerDiscovery = (function () {
-  const val = process.env.COCALC_CONAT_PEER_DISCOVERY ?? "KUBECTL";
-  if (!isPeerDiscovery(val)) {
-    throw Error(`Invalid COCALC_CONAT_PEER_DISCOVERY: ${val}`);
-  }
-  return val;
-})();
+export type PodInfos = { name: string; podIP: string }[];
 
 const logger = getLogger("conat:socketio:dns-scan");
 
@@ -100,49 +86,11 @@ export async function getAddresses(): Promise<string[]> {
   const i = h.lastIndexOf("-");
   const prefix = h.slice(0, i);
 
-  const podInfos = await getPodInfos();
+  const podInfos: PodInfos = await getAddressesFromK8sApi();
   for (const { name, podIP } of podInfos) {
     if (name != h && name.startsWith(prefix)) {
       v.push(`http://${podIP}:${port}`);
     }
   }
   return v;
 }
-
-async function getPodInfos(): Promise<{ name: string; podIP: string }[]> {
-  switch (PEER_DISCOVERY) {
-    case "KUBECTL":
-      return await getAddressesFromKubectl();
-    case "API":
-      return await getAddressesFromK8sApi();
-    default:
-      unreachable(PEER_DISCOVERY);
-      throw Error(`Unknown PEER_DISCOVERY: ${PEER_DISCOVERY}`);
-  }
-}
-
-async function getAddressesFromKubectl(): Promise<
-  { name: string; podIP: string }[]
-> {
-  const ret: { name: string; podIP: string }[] = [];
-  const { stdout } = await executeCode({
-    command: "kubectl",
-    args: [
-      "get",
-      "pods",
-      "-l",
-      "run=hub-conat-router",
-      "-o",
-      `jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.status.podIP}{"\\n"}{end}`,
-    ],
-  });
-  for (const x of stdout.split("\n")) {
-    const row = split(x);
-    if (row.length == 2) {
-      ret.push({ name: row[0], podIP: row[1] });
-    } else {
-      logger.warn(`Unexpected row from kubectl: ${x}`);
-    }
-  }
-  return ret;
-}