From c9d8dc88623d9de5e399482bfce4aaaae5c5321b Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Fri, 18 Apr 2025 15:36:20 +0545 Subject: [PATCH 01/13] initial-try --- packages/docprovider/src/ydrive.ts | 37 +++++++++++++++++++ packages/docprovider/src/yprovider.ts | 1 + .../jupyter_server_ydoc/app.py | 1 + .../jupyter_server_ydoc/handlers.py | 24 ++++++++++-- .../jupyter_server_ydoc/loaders.py | 2 + .../jupyter_server_ydoc/rooms.py | 8 ++++ .../jupyter_server_ydoc/utils.py | 1 + 7 files changed, 71 insertions(+), 3 deletions(-) diff --git a/packages/docprovider/src/ydrive.ts b/packages/docprovider/src/ydrive.ts index 20e5db4f..85b6812b 100644 --- a/packages/docprovider/src/ydrive.ts +++ b/packages/docprovider/src/ydrive.ts @@ -1,6 +1,8 @@ // Copyright (c) Jupyter Development Team. // Distributed under the terms of the Modified BSD License. +console.log('Inside ydrive!'); + import { PageConfig, URLExt } from '@jupyterlab/coreutils'; import { TranslationBundle } from '@jupyterlab/translation'; import { @@ -20,7 +22,11 @@ import { IDocumentProvider, ISharedModelFactory } from '@jupyter/collaborative-drive'; +import { messageSync } from 'y-websocket'; import { Awareness } from 'y-protocols/awareness'; +import { encodeStateAsUpdate } from 'yjs'; +import * as syncProtocol from 'y-protocols/sync'; +import { writeVarUint, createEncoder, toUint8Array } from 'lib0/encoding'; const DISABLE_RTC = PageConfig.getOption('disableRTC') === 'true' ? true : false; @@ -50,6 +56,7 @@ export class RtcContentProvider implements IContentProvider { constructor(options: RtcContentProvider.IOptions) { + console.log('RTC content provider activated!'); super(options); this._user = options.user; this._trans = options.trans; @@ -81,6 +88,7 @@ export class RtcContentProvider localPath: string, options?: Contents.IFetchOptions ): Promise { + console.log('Wow, get is being called!'); if (options && options.format && options.type) { const key = `${options.format}:${options.type}:${localPath}`; const provider = this._providers.get(key); @@ -118,12 +126,40 @@ export class RtcContentProvider options: Partial = {} ): Promise { // Check that there is a provider - it won't e.g. if the document model is not collaborative. + console.log("Wow inside ydrive's save!", localPath, options); if (options.format && options.type) { const key = `${options.format}:${options.type}:${localPath}`; const provider = this._providers.get(key); if (provider) { + // // Gpt thing + // // Send a "save" opcode (e.g., MessageType.DKP_SAVE = 100) with the encoded state + // const update = encodeStateAsUpdate(provider.wsProvider?.doc!); + + // const payload = new Uint8Array(1 + update.length); + // payload[0] = 0; // your custom opcode for save + // payload.set(update, 1); + // // gpt gave this, it's working fine but server is not sure about payload. So I might need to + // // format playload differently + // // after server starts recognizing saves + // // I need to do below, if above task seems very diffcult, i might first finish below's + // // bind autosave settings to file, and send it also + // // now see handlers.py line 289 + + // provider.wsProvider?.ws?.send(payload); + const update = encodeStateAsUpdate(provider.wsProvider?.doc!); + + const encoder = createEncoder(); + writeVarUint(encoder, messageSync); // → 0 = "sync channel" + syncProtocol.writeUpdate(encoder, update); // → writes subtype=2 + payload + const msg = toUint8Array(encoder); + console.log('First bytes: ', msg[0], msg[1]); + + // 3. fire‑and‑forget + provider.wsProvider?.ws?.send(msg); + // Save is done from the backend + // provider.wsProvider?.ws?.send('0'); const fetchOptions: Contents.IFetchOptions = { type: options.type, format: options.format, @@ -147,6 +183,7 @@ export class RtcContentProvider options: Contents.ISharedFactoryOptions, sharedModel: YDocument ) => { + console.log('On create!'); if (typeof options.format !== 'string') { return; } diff --git a/packages/docprovider/src/yprovider.ts b/packages/docprovider/src/yprovider.ts index e02a3463..15605984 100644 --- a/packages/docprovider/src/yprovider.ts +++ b/packages/docprovider/src/yprovider.ts @@ -160,6 +160,7 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { }; private _onSync = (isSynced: boolean) => { + console.log('on _onSync'); if (isSynced) { if (this._yWebsocketProvider) { this._yWebsocketProvider.off('sync', this._onSync); diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index b49ee7af..e72424cc 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -204,6 +204,7 @@ async def get_document( if isinstance(room, DocumentRoom): if copy: update = room.ydoc.get_update() + print("This is the update received!", update) fork_ydoc = Doc() fork_ydoc.apply_update(update) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index 63691d91..3b768d48 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -274,6 +274,7 @@ async def send(self, message): Send a message to the client. """ # needed to be compatible with WebsocketServer (websocket.send) + print("\nYocWebsockethandler is send a message!\n") try: self.write_message(message, binary=True) except Exception as e: @@ -290,8 +291,21 @@ async def on_message(self, message): """ On message receive. """ - message_type = message[0] - + m = message[0] + # Here I am getting sync message. + # But payload seems to be incorrect. + # My thought is that if autosave is off and users manually saves there must be diff. + # So we sync, but i need to figure out the payload's style + message_type = m + if m == MessageType.CHAT: + print("Chat") + elif m == MessageType.SYNC: + print("Sync") + elif m == MessageType.AWARENESS: + print("Awarness") + elif m == MessageType.SAVE: + print("Save") + print(message_type) if message_type == MessageType.CHAT: msg = message[2:].decode("utf-8") @@ -320,11 +334,14 @@ def on_close(self) -> None: On connection close. """ # stop serving this client + print("\n\n", "Websocket is closed, ", self.room.clients, self, "\n\n") self._message_queue.put_nowait(b"") - if isinstance(self.room, DocumentRoom) and self.room.clients == [self]: + print("conditions->", isinstance(self.room, DocumentRoom), self.room.clients == {self}) + if isinstance(self.room, DocumentRoom) and self.room.clients == {self}: # no client in this room after we disconnect # keep the document for a while in case someone reconnects self.log.info("Cleaning room: %s", self._room_id) + print("\n\nTask created!\n\n") self.room.cleaner = asyncio.create_task(self._clean_room()) if self._room_id != "JupyterLab:globalAwareness": self._emit_awareness_event(self.current_user.username, "leave") @@ -360,6 +377,7 @@ async def _clean_room(self) -> None: contains a copy of the document. In addition, we remove the file if there is no rooms subscribed to it. """ + print("\nCleaning the room after delay of: ", self._cleanup_delay, "\n") assert isinstance(self.room, DocumentRoom) if self._cleanup_delay is None: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py index 9df2bea6..1e3ced51 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py @@ -141,6 +141,7 @@ async def maybe_save_content(self, model: dict[str, Any]) -> dict[str, Any] | No ### Note: If there is changes on disk, this method will raise an OutOfBandChanges exception. """ + print("\nTrying to save ", self.path) async with self._lock: path = self.path if model["type"] not in {"directory", "file", "notebook"}: @@ -217,6 +218,7 @@ async def maybe_notify(self) -> None: """ Notifies subscribed rooms about out-of-band file changes. """ + print("\n", "len(subscriptions): ", self.number_of_subscriptions, "\n") do_notify = False filepath_change = False async with self._lock: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 0ebc6935..dbe2d0cd 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -41,6 +41,13 @@ def __init__( self._file_format: str = file_format self._file_type: str = file_type self._file: FileLoader = file + + async def testing(): + while True: + await asyncio.sleep(2) + print("Clients length: ", self.clients) + + asyncio.create_task(testing()) self._document = YDOCS.get(self._file_type, YFILE)(self.ydoc, self.awareness) self._document.path = self._file.path @@ -247,6 +254,7 @@ def _on_document_change(self, target: str, event: Any) -> None: document. This tasks are debounced (60 seconds by default) so we need to cancel previous tasks before creating a new one. """ + print("Document change detected", target, event) if self._update_lock.locked(): return diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py index 22c51d87..0eacd6f1 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py @@ -20,6 +20,7 @@ class MessageType(IntEnum): SYNC = 0 AWARENESS = 1 CHAT = 125 + SAVE = 100 class LogLevel(Enum): From 789c87475547e3eb14799550b846cfa9d5786e5d Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Fri, 25 Apr 2025 23:55:46 +0545 Subject: [PATCH 02/13] prevent-always-autosave --- .../src/collaboration.ts | 1 + .../docprovider-extension/src/filebrowser.ts | 15 +++-- packages/docprovider/src/ydrive.ts | 56 ++++++------------- packages/docprovider/src/yprovider.ts | 1 - .../jupyter_server_ydoc/app.py | 1 - .../jupyter_server_ydoc/handlers.py | 14 ----- .../jupyter_server_ydoc/loaders.py | 2 - .../jupyter_server_ydoc/rooms.py | 21 ++++--- .../jupyter_server_ydoc/utils.py | 1 - 9 files changed, 41 insertions(+), 71 deletions(-) diff --git a/packages/collaboration-extension/src/collaboration.ts b/packages/collaboration-extension/src/collaboration.ts index c11d353e..b5c54829 100644 --- a/packages/collaboration-extension/src/collaboration.ts +++ b/packages/collaboration-extension/src/collaboration.ts @@ -21,6 +21,7 @@ import { URLExt } from '@jupyterlab/coreutils'; import { ServerConnection } from '@jupyterlab/services'; import { IStateDB, StateDB } from '@jupyterlab/statedb'; import { ITranslator, nullTranslator } from '@jupyterlab/translation'; +import { ISettingRegistry } from '@jupyterlab/settingregistry'; import { Menu, MenuBar } from '@lumino/widgets'; diff --git a/packages/docprovider-extension/src/filebrowser.ts b/packages/docprovider-extension/src/filebrowser.ts index 4815659c..33289c7f 100644 --- a/packages/docprovider-extension/src/filebrowser.ts +++ b/packages/docprovider-extension/src/filebrowser.ts @@ -55,12 +55,13 @@ export const rtcContentProvider: JupyterFrontEndPlugin { + globalAwareness: Awareness | null, + settingRegistry: ISettingRegistry | null + ): Promise => { const trans = translator.load('jupyter_collaboration'); const defaultDrive = (app.serviceManager.contents as ContentsManager) .defaultDrive; @@ -75,12 +76,16 @@ export const rtcContentProvider: JupyterFrontEndPlugin(); + this._docmanagerSettings = options.docmanagerSettings; } /** @@ -88,7 +84,6 @@ export class RtcContentProvider localPath: string, options?: Contents.IFetchOptions ): Promise { - console.log('Wow, get is being called!'); if (options && options.format && options.type) { const key = `${options.format}:${options.type}:${localPath}`; const provider = this._providers.get(key); @@ -126,40 +121,12 @@ export class RtcContentProvider options: Partial = {} ): Promise { // Check that there is a provider - it won't e.g. if the document model is not collaborative. - console.log("Wow inside ydrive's save!", localPath, options); if (options.format && options.type) { const key = `${options.format}:${options.type}:${localPath}`; const provider = this._providers.get(key); if (provider) { - // // Gpt thing - // // Send a "save" opcode (e.g., MessageType.DKP_SAVE = 100) with the encoded state - // const update = encodeStateAsUpdate(provider.wsProvider?.doc!); - - // const payload = new Uint8Array(1 + update.length); - // payload[0] = 0; // your custom opcode for save - // payload.set(update, 1); - // // gpt gave this, it's working fine but server is not sure about payload. So I might need to - // // format playload differently - // // after server starts recognizing saves - // // I need to do below, if above task seems very diffcult, i might first finish below's - // // bind autosave settings to file, and send it also - // // now see handlers.py line 289 - - // provider.wsProvider?.ws?.send(payload); - const update = encodeStateAsUpdate(provider.wsProvider?.doc!); - - const encoder = createEncoder(); - writeVarUint(encoder, messageSync); // → 0 = "sync channel" - syncProtocol.writeUpdate(encoder, update); // → writes subtype=2 + payload - const msg = toUint8Array(encoder); - console.log('First bytes: ', msg[0], msg[1]); - - // 3. fire‑and‑forget - provider.wsProvider?.ws?.send(msg); - - // Save is done from the backend - // provider.wsProvider?.ws?.send('0'); + provider.wsProvider?.ws?.send('save_to_disc'); const fetchOptions: Contents.IFetchOptions = { type: options.type, format: options.format, @@ -183,10 +150,22 @@ export class RtcContentProvider options: Contents.ISharedFactoryOptions, sharedModel: YDocument ) => { - console.log('On create!'); if (typeof options.format !== 'string') { return; } + // Set initial autosave value, used to determine backend autosave (default: true) + const autosave = + (this._docmanagerSettings?.composite?.['autosave'] as boolean) ?? true; + + sharedModel.awareness.setLocalStateField('autosave', autosave); + + // Watch for changes in settings + this._docmanagerSettings?.changed.connect(() => { + const newAutosave = + (this._docmanagerSettings?.composite?.['autosave'] as boolean) ?? true; + sharedModel.awareness.setLocalStateField('autosave', newAutosave); + }); + try { const provider = new WebSocketProvider({ url: URLExt.join(this._serverSettings.wsUrl, DOCUMENT_PROVIDER_URL), @@ -272,6 +251,7 @@ export class RtcContentProvider private _providers: Map; private _ydriveFileChanged = new Signal(this); private _serverSettings: ServerConnection.ISettings; + private _docmanagerSettings: ISettingRegistry.ISettings | null; } /** diff --git a/packages/docprovider/src/yprovider.ts b/packages/docprovider/src/yprovider.ts index 15605984..e02a3463 100644 --- a/packages/docprovider/src/yprovider.ts +++ b/packages/docprovider/src/yprovider.ts @@ -160,7 +160,6 @@ export class WebSocketProvider implements IDocumentProvider, IForkProvider { }; private _onSync = (isSynced: boolean) => { - console.log('on _onSync'); if (isSynced) { if (this._yWebsocketProvider) { this._yWebsocketProvider.off('sync', this._onSync); diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py index e72424cc..b49ee7af 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py @@ -204,7 +204,6 @@ async def get_document( if isinstance(room, DocumentRoom): if copy: update = room.ydoc.get_update() - print("This is the update received!", update) fork_ydoc = Doc() fork_ydoc.apply_update(update) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index 3b768d48..439a2e82 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -274,7 +274,6 @@ async def send(self, message): Send a message to the client. """ # needed to be compatible with WebsocketServer (websocket.send) - print("\nYocWebsockethandler is send a message!\n") try: self.write_message(message, binary=True) except Exception as e: @@ -297,15 +296,6 @@ async def on_message(self, message): # My thought is that if autosave is off and users manually saves there must be diff. # So we sync, but i need to figure out the payload's style message_type = m - if m == MessageType.CHAT: - print("Chat") - elif m == MessageType.SYNC: - print("Sync") - elif m == MessageType.AWARENESS: - print("Awarness") - elif m == MessageType.SAVE: - print("Save") - print(message_type) if message_type == MessageType.CHAT: msg = message[2:].decode("utf-8") @@ -334,14 +324,11 @@ def on_close(self) -> None: On connection close. """ # stop serving this client - print("\n\n", "Websocket is closed, ", self.room.clients, self, "\n\n") self._message_queue.put_nowait(b"") - print("conditions->", isinstance(self.room, DocumentRoom), self.room.clients == {self}) if isinstance(self.room, DocumentRoom) and self.room.clients == {self}: # no client in this room after we disconnect # keep the document for a while in case someone reconnects self.log.info("Cleaning room: %s", self._room_id) - print("\n\nTask created!\n\n") self.room.cleaner = asyncio.create_task(self._clean_room()) if self._room_id != "JupyterLab:globalAwareness": self._emit_awareness_event(self.current_user.username, "leave") @@ -377,7 +364,6 @@ async def _clean_room(self) -> None: contains a copy of the document. In addition, we remove the file if there is no rooms subscribed to it. """ - print("\nCleaning the room after delay of: ", self._cleanup_delay, "\n") assert isinstance(self.room, DocumentRoom) if self._cleanup_delay is None: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py index 1e3ced51..9df2bea6 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/loaders.py @@ -141,7 +141,6 @@ async def maybe_save_content(self, model: dict[str, Any]) -> dict[str, Any] | No ### Note: If there is changes on disk, this method will raise an OutOfBandChanges exception. """ - print("\nTrying to save ", self.path) async with self._lock: path = self.path if model["type"] not in {"directory", "file", "notebook"}: @@ -218,7 +217,6 @@ async def maybe_notify(self) -> None: """ Notifies subscribed rooms about out-of-band file changes. """ - print("\n", "len(subscriptions): ", self.number_of_subscriptions, "\n") do_notify = False filepath_change = False async with self._lock: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index dbe2d0cd..92d39c92 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -41,13 +41,6 @@ def __init__( self._file_format: str = file_format self._file_type: str = file_type self._file: FileLoader = file - - async def testing(): - while True: - await asyncio.sleep(2) - print("Clients length: ", self.clients) - - asyncio.create_task(testing()) self._document = YDOCS.get(self._file_type, YFILE)(self.ydoc, self.awareness) self._document.path = self._file.path @@ -254,7 +247,18 @@ def _on_document_change(self, target: str, event: Any) -> None: document. This tasks are debounced (60 seconds by default) so we need to cancel previous tasks before creating a new one. """ - print("Document change detected", target, event) + # Collect autosave values from all clients + autosave_states = [ + state.get("autosave", True) + for state in self.awareness.states.values() + if state # skip empty states + ] + + # Enable autosave if at least one client has it turned on + autosave = any(autosave_states) + + if not autosave: + return if self._update_lock.locked(): return @@ -273,7 +277,6 @@ async def _maybe_save_document(self, saving_document: asyncio.Task | None) -> No """ if self._save_delay is None: return - if saving_document is not None and not saving_document.done(): # the document is being saved, cancel that saving_document.cancel() diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py index 0eacd6f1..22c51d87 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py @@ -20,7 +20,6 @@ class MessageType(IntEnum): SYNC = 0 AWARENESS = 1 CHAT = 125 - SAVE = 100 class LogLevel(Enum): From 40fba549c5b3d673706a35ed2603af67d2b9e360 Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Sat, 26 Apr 2025 00:05:19 +0545 Subject: [PATCH 03/13] fix-unused-import --- packages/collaboration-extension/src/collaboration.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/collaboration-extension/src/collaboration.ts b/packages/collaboration-extension/src/collaboration.ts index b5c54829..c11d353e 100644 --- a/packages/collaboration-extension/src/collaboration.ts +++ b/packages/collaboration-extension/src/collaboration.ts @@ -21,7 +21,6 @@ import { URLExt } from '@jupyterlab/coreutils'; import { ServerConnection } from '@jupyterlab/services'; import { IStateDB, StateDB } from '@jupyterlab/statedb'; import { ITranslator, nullTranslator } from '@jupyterlab/translation'; -import { ISettingRegistry } from '@jupyterlab/settingregistry'; import { Menu, MenuBar } from '@lumino/widgets'; From 19a0335ade7b8b427ee5885b03e70a1e1af3e40c Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Sat, 26 Apr 2025 00:13:44 +0545 Subject: [PATCH 04/13] check_for_manual_save --- .../jupyter_server_ydoc/handlers.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index 439a2e82..844431c3 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -290,12 +290,15 @@ async def on_message(self, message): """ On message receive. """ - m = message[0] - # Here I am getting sync message. - # But payload seems to be incorrect. - # My thought is that if autosave is off and users manually saves there must be diff. - # So we sync, but i need to figure out the payload's style - message_type = m + if message == "save_to_disc": + try: + self.room._saving_document = asyncio.create_task( + self.room._maybe_save_document(self.room._saving_document) + ) + except Exception: + self.log.error("Couldn't save content from room: %s", self._room_id) + + message_type = message[0] if message_type == MessageType.CHAT: msg = message[2:].decode("utf-8") From 45b9ec2ebb5e9fe2d6811757c211c84825392946 Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Sat, 26 Apr 2025 00:49:40 +0545 Subject: [PATCH 05/13] force-autosave-by-default --- packages/docprovider-extension/src/filebrowser.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/docprovider-extension/src/filebrowser.ts b/packages/docprovider-extension/src/filebrowser.ts index 33289c7f..cf384d8a 100644 --- a/packages/docprovider-extension/src/filebrowser.ts +++ b/packages/docprovider-extension/src/filebrowser.ts @@ -79,6 +79,11 @@ export const rtcContentProvider: JupyterFrontEndPlugin Date: Sat, 26 Apr 2025 16:36:18 +0545 Subject: [PATCH 06/13] remove-force-autosave --- packages/docprovider-extension/src/filebrowser.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/docprovider-extension/src/filebrowser.ts b/packages/docprovider-extension/src/filebrowser.ts index cf384d8a..3c5a3a7e 100644 --- a/packages/docprovider-extension/src/filebrowser.ts +++ b/packages/docprovider-extension/src/filebrowser.ts @@ -80,10 +80,6 @@ export const rtcContentProvider: JupyterFrontEndPlugin Date: Sat, 26 Apr 2025 16:53:03 +0545 Subject: [PATCH 07/13] fix-type-hints --- .../jupyter-server-ydoc/jupyter_server_ydoc/handlers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index c8ca519d..f0cdfa32 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -9,6 +9,7 @@ from logging import Logger from typing import Any from uuid import uuid4 +from typing import cast from jupyter_server.auth import authorized from jupyter_server.base.handlers import APIHandler, JupyterHandler @@ -293,8 +294,9 @@ async def on_message(self, message): """ if message == "save_to_disc": try: - self.room._saving_document = asyncio.create_task( - self.room._maybe_save_document(self.room._saving_document) + room = cast(DocumentRoom, self.room) + room._saving_document = asyncio.create_task( + room._maybe_save_document(room._saving_document) ) except Exception: self.log.error("Couldn't save content from room: %s", self._room_id) From 72fd92b97a047a1ead46f44f6ea99c9f788e0678 Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Sat, 26 Apr 2025 17:12:17 +0545 Subject: [PATCH 08/13] fix-autosave-in-tests --- projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 75700b5d..56113749 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -254,6 +254,10 @@ def _on_document_change(self, target: str, event: Any) -> None: if state # skip empty states ] + # If no states exist (e.g., during tests), force autosave to be True + if not autosave_states: + autosave_states = [True] + # Enable autosave if at least one client has it turned on autosave = any(autosave_states) From 7e84bf8c1e492a77390c13456f7b4037aa40e5dd Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Sat, 26 Apr 2025 17:30:30 +0545 Subject: [PATCH 09/13] add-tests --- .../jupyter_server_ydoc/pytest_plugin.py | 2 +- tests/test_rooms.py | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py index 75f18448..a41d69d7 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/pytest_plugin.py @@ -276,7 +276,7 @@ def _inner( last_modified: datetime | None = None, save_delay: float | None = None, store: SQLiteYStore | None = None, - writable: bool = False, + writable: bool = True, ) -> tuple[FakeContentsManager, FileLoader, DocumentRoom]: paths = {id: path} diff --git a/tests/test_rooms.py b/tests/test_rooms.py index 3593094a..f78176c3 100644 --- a/tests/test_rooms.py +++ b/tests/test_rooms.py @@ -51,9 +51,7 @@ async def test_defined_save_delay_should_save_content_after_document_change( rtc_create_mock_document_room, ): content = "test" - cm, _, room = rtc_create_mock_document_room( - "test-id", "test.txt", content, save_delay=0.01, writable=True - ) + cm, _, room = rtc_create_mock_document_room("test-id", "test.txt", content, save_delay=0.01) await room.initialize() room._document.source = "Test 2" @@ -79,6 +77,50 @@ async def test_undefined_save_delay_should_not_save_content_after_document_chang assert "save" not in cm.actions +async def test_should_not_save_content_when_all_clients_have_autosave_disabled( + rtc_create_mock_document_room, +): + content = "test" + cm, _, room = rtc_create_mock_document_room("test-id", "test.txt", content, save_delay=0.01) + + # Disable autosave for all existing clients + for state in room.awareness._states.values(): + if state is not None: + state["autosave"] = False + + # Inject a dummy client with autosave disabled + room.awareness._states[9999] = {"autosave": False} + + await room.initialize() + room._document.source = "Test 2" + + await asyncio.sleep(0.15) + + assert "save" not in cm.actions + + +async def test_should_save_content_when_at_least_one_client_has_autosave_enabled( + rtc_create_mock_document_room, +): + content = "test" + cm, _, room = rtc_create_mock_document_room("test-id", "test.txt", content, save_delay=0.01) + + # Disable autosave for all existing clients + for state in room.awareness._states.values(): + if state is not None: + state["autosave"] = False + + # Inject a dummy client with autosave enabled + room.awareness._states[10000] = {"autosave": True} + + await room.initialize() + room._document.source = "Test 2" + + await asyncio.sleep(0.15) + + assert "save" in cm.actions + + # The following test should be restored when package versions are fixed. # async def test_document_path(rtc_create_mock_document_room): From 6b94adb13b9e5312c85e1095828369ae7870a79b Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Tue, 6 May 2025 21:11:18 +0545 Subject: [PATCH 10/13] follow-y-protocol --- packages/docprovider/src/ydrive.ts | 10 +++++++++- .../jupyter_server_ydoc/handlers.py | 20 ++++++++++--------- .../jupyter_server_ydoc/rooms.py | 11 ++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/docprovider/src/ydrive.ts b/packages/docprovider/src/ydrive.ts index 00e3bb9f..a905d9d7 100644 --- a/packages/docprovider/src/ydrive.ts +++ b/packages/docprovider/src/ydrive.ts @@ -22,10 +22,18 @@ import { } from '@jupyter/collaborative-drive'; import { Awareness } from 'y-protocols/awareness'; import { ISettingRegistry } from '@jupyterlab/settingregistry'; +import * as encoding from 'lib0/encoding'; const DISABLE_RTC = PageConfig.getOption('disableRTC') === 'true' ? true : false; +const SAVE_MESSAGE = (() => { + const encoder = encoding.createEncoder(); + encoding.writeVarUint(encoder, 2); + encoding.writeVarString(encoder, 'save'); + return encoding.toUint8Array(encoder); +})(); + /** * The url for the default drive service. */ @@ -126,7 +134,7 @@ export class RtcContentProvider const provider = this._providers.get(key); if (provider) { - provider.wsProvider?.ws?.send('save_to_disc'); + provider.wsProvider?.ws?.send(SAVE_MESSAGE); const fetchOptions: Contents.IFetchOptions = { type: options.type, format: options.format, diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index f0cdfa32..efa42db5 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -33,6 +33,7 @@ room_id_from_encoded_path, ) from .websocketserver import JupyterWebsocketServer, RoomNotFound +from .utils import MessageType YFILE = YDOCS["file"] @@ -292,15 +293,16 @@ async def on_message(self, message): """ On message receive. """ - if message == "save_to_disc": - try: - room = cast(DocumentRoom, self.room) - room._saving_document = asyncio.create_task( - room._maybe_save_document(room._saving_document) - ) - except Exception: - self.log.error("Couldn't save content from room: %s", self._room_id) - return + message_type = message[0] + if message_type == MessageType.UPDATE: + msg = message[2:].decode("utf-8") + if msg == "save": + try: + room = cast(DocumentRoom, self.room) + room._save_to_disc() + except Exception: + self.log.error("Couldn't save content from room: %s", self._room_id) + return self._message_queue.put_nowait(message) self._websocket_server.ypatch_nb += 1 diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py index 56113749..851293f1 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py @@ -270,6 +270,17 @@ def _on_document_change(self, target: str, event: Any) -> None: self._maybe_save_document(self._saving_document) ) + def _save_to_disc(self): + """ + Called when manual save is triggered. Helpful when autosave is turned off. + """ + if self._update_lock.locked(): + return + + self._saving_document = asyncio.create_task( + self._maybe_save_document(self._saving_document) + ) + async def _maybe_save_document(self, saving_document: asyncio.Task | None) -> None: """ Saves the content of the document to disk. From 9ddd24d4511d93b29c6ca9f079cb2f3364df2eca Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Tue, 6 May 2025 21:23:36 +0545 Subject: [PATCH 11/13] add-update-value --- projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py index 22c51d87..12fb836d 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py @@ -19,6 +19,7 @@ class MessageType(IntEnum): SYNC = 0 AWARENESS = 1 + UPDATE = 2 CHAT = 125 From cd23070f055a722a88f6fd8754e7c6a84fab2ed6 Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Wed, 21 May 2025 21:03:33 +0545 Subject: [PATCH 12/13] refactor-code --- packages/docprovider/src/ydrive.ts | 4 +++- projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py | 2 +- projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/docprovider/src/ydrive.ts b/packages/docprovider/src/ydrive.ts index a905d9d7..3504f702 100644 --- a/packages/docprovider/src/ydrive.ts +++ b/packages/docprovider/src/ydrive.ts @@ -27,9 +27,11 @@ import * as encoding from 'lib0/encoding'; const DISABLE_RTC = PageConfig.getOption('disableRTC') === 'true' ? true : false; +const RAW_MESSAGE_TYPE = 2; + const SAVE_MESSAGE = (() => { const encoder = encoding.createEncoder(); - encoding.writeVarUint(encoder, 2); + encoding.writeVarUint(encoder, RAW_MESSAGE_TYPE); encoding.writeVarString(encoder, 'save'); return encoding.toUint8Array(encoder); })(); diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index efa42db5..9a0f1d79 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -294,7 +294,7 @@ async def on_message(self, message): On message receive. """ message_type = message[0] - if message_type == MessageType.UPDATE: + if message_type == MessageType.RAW: msg = message[2:].decode("utf-8") if msg == "save": try: diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py index 12fb836d..6d866799 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/utils.py @@ -19,7 +19,7 @@ class MessageType(IntEnum): SYNC = 0 AWARENESS = 1 - UPDATE = 2 + RAW = 2 CHAT = 125 From 518fec47e7efc642df9a8097b1b54a3df32b0760 Mon Sep 17 00:00:00 2001 From: Darshan808 Date: Fri, 23 May 2025 08:44:45 +0545 Subject: [PATCH 13/13] use-decoder --- .../jupyter_server_ydoc/handlers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py index 9a0f1d79..940370a0 100644 --- a/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py +++ b/projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py @@ -34,6 +34,7 @@ ) from .websocketserver import JupyterWebsocketServer, RoomNotFound from .utils import MessageType +from pycrdt import Decoder YFILE = YDOCS["file"] @@ -293,16 +294,17 @@ async def on_message(self, message): """ On message receive. """ - message_type = message[0] - if message_type == MessageType.RAW: - msg = message[2:].decode("utf-8") + decoder = Decoder(message) + header = decoder.read_var_uint() + if header == MessageType.RAW: + msg = decoder.read_var_string() if msg == "save": try: room = cast(DocumentRoom, self.room) room._save_to_disc() except Exception: self.log.error("Couldn't save content from room: %s", self._room_id) - return + return self._message_queue.put_nowait(message) self._websocket_server.ypatch_nb += 1