Merge remote-tracking branch 'origin/master' into jupyter-llm-8500

Author: haraldschilly

src/.claude/settings.json

@@ -1,33 +1,39 @@
 {
   "permissions": {
     "allow": [
+      "Bash(../node_modules/.bin/tsc:*)",
+      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
+      "Bash(curl:*)",
+      "Bash(docker run:*)",
       "Bash(find:*)",
       "Bash(gh pr view:*)",
       "Bash(gh:*)",
       "Bash(git add:*)",
+      "Bash(git checkout:*)",
       "Bash(git commit:*)",
+      "Bash(git push:*)",
       "Bash(grep:*)",
       "Bash(node:*)",
       "Bash(npm show:*)",
+      "Bash(npm view:*)",
       "Bash(npx tsc:*)",
       "Bash(pnpm build:*)",
+      "Bash(pnpm i18n:*)",
       "Bash(pnpm ts-build:*)",
       "Bash(pnpm tsc:*)",
       "Bash(prettier -w:*)",
+      "Bash(psql:*)",
       "WebFetch(domain:cocalc.com)",
       "WebFetch(domain:doc.cocalc.com)",
       "WebFetch(domain:docs.anthropic.com)",
       "WebFetch(domain:github.com)",
-      "Bash(git checkout:*)",
-      "Bash(git push:*)",
-      "Bash(NODE_OPTIONS=--max-old-space-size=8192 ../node_modules/.bin/tsc --noEmit)",
-      "Bash(docker run:*)",
-      "Bash(../node_modules/.bin/tsc:*)",
-      "Bash(npm view:*)",
-      "WebFetch(domain:www.anthropic.com)",
       "WebFetch(domain:mistral.ai)",
-      "Bash(pnpm i18n:*)",
-      "WebFetch(domain:simplelocalize.io)"
+      "WebFetch(domain:simplelocalize.io)",
+      "Bash(pnpm list:*)",
+      "Bash(pnpm why:*)",
+      "mcp__github__get_issue",
+      "WebFetch(domain:www.anthropic.com)",
+      "mcp__cclsp__find_definition"
     ],
     "deny": []
   }

src/CLAUDE.md

@@ -73,6 +73,8 @@ CoCalc is organized as a monorepo with key packages:
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+- **Authentication Waiting**: When frontend code needs to wait for user authentication, use `redux.getStore("account").async_wait({ until: () => store.get_account_id() != null, timeout: 0 })` to wait indefinitely until authentication completes
+- **Conat DKV Usage**: For key-value storage with real-time sync, use `webapp_client.conat_client.dkv({ account_id, name: "store-name" })` to get a distributed key-value store that syncs across sessions
 
 #### Backend Architecture
 
@@ -81,6 +83,8 @@ CoCalc is organized as a monorepo with key packages:
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
+- **Database Access**: Use `getPool()` from `@cocalc/database/pool` for direct database queries in hub/backend code. Example: `const pool = getPool(); const { rows } = await pool.query('SELECT * FROM table WHERE id = $1', [id]);`
+- **Hub Migration Functions**: Migration functions in hub should be designed to run once at startup, use batch processing with delays between batches to avoid database saturation
 
 #### Communication Patterns
 

src/packages/conat/service/typed.ts

@@ -9,7 +9,7 @@ import type { Options, ServiceCall } from "./service";
 export type { ConatService };
 
 export interface Extra {
-  ping: typeof pingConatService;
+  ping: (opts?: { maxWait?: number }) => Promise<void>;
   waitFor: (opts?: { maxWait?: number }) => Promise<void>;
 }
 

src/packages/database/postgres/news.ts

@@ -23,11 +23,13 @@ export function clearCache(): void {
 const Q_FEED = `
 SELECT
   id, channel, title, text, url,
-  extract(epoch from date::timestamp)::integer as date
+  extract(epoch from date::timestamp)::integer as date,
+  extract(epoch from until::timestamp)::integer as until
 FROM news
 WHERE news.date <= NOW()
   AND hide IS NOT TRUE
   AND channel != '${EVENT_CHANNEL}'
+  AND (until IS NULL OR until > NOW())
 ORDER BY date DESC
 LIMIT 100`;
 
@@ -40,7 +42,8 @@ export async function getFeedData(): Promise<NewsItem[]> {
 const Q_BY_ID = `
 SELECT
   id, channel, title, text, url, hide, tags,
-  extract(epoch from date::timestamptz)::INTEGER as date
+  extract(epoch from date::timestamptz)::INTEGER as date,
+  extract(epoch from until::timestamptz)::INTEGER as until
 FROM news
 WHERE id = $1`;
 
@@ -56,7 +59,9 @@ const Q_BY_ID_USER = `
 SELECT
   id, channel, title, text, url, hide, tags, history,
   date >= NOW() as future,
-  extract(epoch from date::timestamptz)::INTEGER as date
+  until IS NOT NULL AND until <= NOW() as expired,
+  extract(epoch from date::timestamptz)::INTEGER as date,
+  extract(epoch from until::timestamptz)::INTEGER as until
 FROM news
 WHERE id = $1`;
 
@@ -68,6 +73,7 @@ WHERE date >= (SELECT date FROM news WHERE id = $1)
   AND hide IS NOT TRUE
   AND date < NOW()
   AND channel != '${EVENT_CHANNEL}'
+  AND (until IS NULL OR until > NOW())
 ORDER BY date ASC, id ASC
 LIMIT 1`;
 
@@ -79,6 +85,7 @@ WHERE date <= (SELECT date FROM news WHERE id = $1)
   AND hide IS NOT TRUE
   AND date < NOW()
   AND channel != '${EVENT_CHANNEL}'
+  AND (until IS NULL OR until > NOW())
 ORDER BY date DESC, id DESC
 LIMIT 1`;
 
@@ -104,7 +111,9 @@ const Q_INDEX = `
 SELECT
   id, channel, title, text, url, hide, tags,
   date >= NOW() as future,
-  extract(epoch from date::timestamptz)::INTEGER as date
+  until IS NOT NULL AND until <= NOW() as expired,
+  extract(epoch from date::timestamptz)::INTEGER as date,
+  extract(epoch from until::timestamptz)::INTEGER as until
 FROM news
     WHERE channel <> '${EVENT_CHANNEL}'
 ORDER BY date DESC
@@ -122,11 +131,13 @@ export async function getIndex(
 const Q_MOST_RECENT = `
 SELECT
   id, channel, title, tags,
-  extract(epoch from date::timestamptz)::INTEGER as date
+  extract(epoch from date::timestamptz)::INTEGER as date,
+  extract(epoch from until::timestamptz)::INTEGER as until
 FROM news
 WHERE date <= NOW()
   AND hide IS NOT TRUE
   AND channel != '${EVENT_CHANNEL}'
+  AND (until IS NULL OR until > NOW())
 ORDER BY date DESC
 LIMIT 1`;
 
@@ -137,11 +148,13 @@ export async function getMostRecentNews(): Promise<RecentHeadline | null> {
 const Q_RECENT = `
 SELECT
   id, channel, title, tags,
-  extract(epoch from date::timestamptz)::INTEGER as date
+  extract(epoch from date::timestamptz)::INTEGER as date,
+  extract(epoch from until::timestamptz)::INTEGER as until
 FROM news
 WHERE date <= NOW()
   AND channel != '${EVENT_CHANNEL}'
   AND hide IS NOT TRUE
+  AND (until IS NULL OR until > NOW())
 ORDER BY date DESC
 LIMIT $1`;
 
@@ -158,11 +171,13 @@ export async function getRecentHeadlines(
 const Q_UPCOMING_NEWS_CHANNEL_ITEMS = `
 SELECT
   id, channel, title, text, url, tags,
-  extract(epoch from date::timestamp)::integer as date
+  extract(epoch from date::timestamp)::integer as date,
+  extract(epoch from until::timestamp)::integer as until
 FROM news
 WHERE date >= NOW()
   AND channel = $1
   AND hide IS NOT TRUE
+  AND (until IS NULL OR until > NOW())
 ORDER BY date
 LIMIT 100`;
 
@@ -176,11 +191,13 @@ export async function getUpcomingNewsChannelItems(
 const Q_PAST_NEWS_CHANNEL_ITEMS = `
 SELECT
   id, channel, title, text, url, tags,
-  extract(epoch from date::timestamp)::integer as date
+  extract(epoch from date::timestamp)::integer as date,
+  extract(epoch from until::timestamp)::integer as until
 FROM news
 WHERE date <= NOW()
   AND channel = $1
   AND hide IS NOT TRUE
+  AND (until IS NULL OR until > NOW())
 ORDER BY date DESC
 LIMIT 100`;