feat: swift newfile change for server

Author: SolaWing

compile_database.py

@@ -165,6 +165,8 @@ def findSwiftModuleRoot(filename):
 
     return (directory, flagFile, compileFile)
 
+def filekey(filename):
+    return os.path.realpath(filename).lower()
 
 class CompileFileInfo:
     def __init__(self, compileFile, store):
@@ -182,26 +184,24 @@ def __init__(self, compileFile, store):
                 if not command:
                     continue
                 if files := i.get("files"):  # batch files, eg: swift module
-                    self.file_info.update((self.key(f), command) for f in files)
+                    self.file_info.update((filekey(f), command) for f in files)
                 if fileLists := i.get(
                     "fileLists"
                 ):  # file list store in a dedicated file
                     self.file_info.update(
-                        (self.key(f), command)
+                        (filekey(f), command)
                         for l in fileLists
                         if os.path.isfile(l)
                         for f in getFileArgs(l, store.setdefault("filelist", {}))
                     )
                 if file := i.get("file"):  # single file info
-                    self.file_info[self.key(file)] = command
+                    self.file_info[filekey(file)] = command
 
     def get(self, filename):
         if command := self.file_info.get(filename.lower()):
+            # xcode 12 escape =, but not recognized...
             return command.replace("\\=", "=")
 
-    def key(self, filename):
-        return os.path.realpath(filename).lower()
-
     def groupby_dir(self) -> dict[str, set[str]]:
         if self.dir_info is None:  # lazy index dir and cmd
             self.dir_info = defaultdict(set)
@@ -213,6 +213,7 @@ def groupby_dir(self) -> dict[str, set[str]]:
         return self.dir_info
 
     # hack new file into current compile file
+    # return: set of filekey for match for file. or None if new_file can't be infered
     def new_file(self, filename):
         # Currently only processing swift files
         if not filename.endswith(".swift"):
@@ -221,7 +222,7 @@ def new_file(self, filename):
         filename = os.path.realpath(filename)
         filename_key = filename.lower()
         if filename_key in self.file_info:
-            return  # already handled
+            return {filename_key}  # already handled
 
         dir = os.path.dirname(filename_key)
         samefile = next(
@@ -234,7 +235,7 @@ def new_file(self, filename):
         cmd_match = next(cmd_split_pattern.finditer(command), None)
         if not cmd_match:
             return
-        assert self.cmd_info
+        assert self.cmd_info # init in groupby_dir
         module_files = self.cmd_info.pop(command)
         index = cmd_match.end()
         from shlex import quote
@@ -247,12 +248,22 @@ def new_file(self, filename):
         self.cmd_info[command] = module_files
         for v in module_files:
             self.file_info[v] = command
+        return module_files
 
+def newfileForCompileFile(filename, compileFile, store) -> set[str] | None:
+    info = compileFileInfoFromStore(compileFile, store)
+    return info.new_file(filename)
 
 def commandForFile(filename, compileFile, store: Dict):
     """
     command = store["compile"][<compileFile>][filename]
+    > will lazy build compile_file info
     """
+    info = compileFileInfoFromStore(compileFile, store)
+    return info.get(filename)
+
+
+def compileFileInfoFromStore(compileFile, store: Dict):
     compile_store = store.setdefault("compile", {})
     info: CompileFileInfo = compile_store.get(compileFile)
     if info is None:  # load {filename.lower: command} dict
@@ -263,34 +274,29 @@ def commandForFile(filename, compileFile, store: Dict):
         # if has additional new_file, generate command for it
         for file in store.get("additional_files") or ():
             info.new_file(file)
-
-    # xcode 12 escape =, but not recognized...
-    return info.get(filename)
+    return info
 
 
 def GetFlagsInCompile(filename, compileFile, store):
-    """read flags from compileFile"""
+    """read flags from compileFile. filename should be realpath"""
     if compileFile:
         command = commandForFile(filename, compileFile, store)
         if command:
             flags = cmd_split(command)[1:]  # ignore executable
             return list(filterFlags(flags, store.setdefault("filelist", {})))
 
 
-def GetFlags(filename: str, compileFile=None, **kwargs):
+def GetFlags(filename: str, compileFile=None, store=None):
     """sourcekit entry function"""
     # NOTE: use store to ensure toplevel storage. child store should be other name
     # see store.setdefault to get all child attributes
-    store = kwargs.get("store", globalStore)
-    filename = os.path.realpath(filename)
+    if not store:
+        store = globalStore
 
     if compileFile:
         if final_flags := GetFlagsInCompile(filename, compileFile, store):
-            return {"flags": final_flags, "do_cache": True}
-
-    if filename.endswith(".swift"):
-        return InferFlagsForSwift(filename, compileFile, store)
-    return {"flags": [], "do_cache": False}
+            return final_flags
+    return None
 
 
 # TODO: c family infer flags #
@@ -330,4 +336,4 @@ def InferFlagsForSwift(filename, compileFile, store):
             "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/",
         ]
 
-    return {"flags": final_flags, "do_cache": True}
+    return final_flags

config/__init__.py

@@ -1 +1,2 @@
 from .config import ServerConfig
+from .env import env

config/env.py

@@ -0,0 +1,22 @@
+# use to get environ config
+from functools import cache
+import os
+
+class Env:
+    def on(self, value: str):
+        try:
+            c = value[0]
+            if c.isdigit(): return bool(int(c))
+            return c in ["t", "T", "y", "Y"]
+        except Exception:
+            return False
+
+    @property
+    @cache
+    def new_file(self):
+        if newfile := os.environ.get("XBS_FEAT_NEWFILE"):
+            return self.on(newfile)
+        return False
+
+env = Env()
+

server.py

@@ -8,12 +8,18 @@
 from typing import Optional
 import urllib.parse
 
-from compile_database import GetFlags
-from config import ServerConfig
+from compile_database import (
+    GetFlags,
+    InferFlagsForSwift,
+    filekey,
+    newfileForCompileFile,
+)
+from config import ServerConfig, env
 from misc import force_remove, get_mtime
 
 logger = logging.getLogger(__name__)
 
+
 def send(data):
     data_str = json.dumps(data)
     logger.debug("Res <-- %s", data_str)
@@ -32,6 +38,11 @@ def uri2filepath(uri):
     return urllib.parse.unquote(result.path)
 
 
+def uri2realpath(uri):
+    path = uri2filepath(uri)
+    return os.path.realpath(path)
+
+
 def uptodate(target: str, srcs: list[str]):
     target_mtime = get_mtime(target)
     srcs_mtime = (get_mtime(src) for src in srcs)
@@ -53,12 +64,14 @@ def __init__(self, root_path: str, cache_path):
         self.observed_uri = set()
         # background thread to observe changes
         self.observed_thread: Optional[Thread] = None
+
         # {path: mtime} cache. use to find changes
         self.observed_info = {self.config.path: get_mtime(self.config.path)}
 
         self.reinit_compile_info()
-        # NOTE:thread-safety: for state shared by main and background thread,
-        # can only changed in sync_compile_file
+        # NOTE:thread-safety: for state shared by main and background watch thread,
+        # can only changed in sync_compile_file, which block all thread and no one access it.
+        # other time, the shared state is readonly and safe..
 
     def get_compile_file(self, config):
         # isolate xcode generate compile file and manual compile_file
@@ -75,7 +88,7 @@ def reinit_compile_info(self):
         """all the compile information may change in background"""
 
         # store use to save compile_datainfo. it will be reload when config changes.
-        self.store = {} # main-thread
+        self.store = {}  # main-thread
         self._compile_file = self.get_compile_file(self.config)
         if os.path.exists(self._compile_file):
             self.compile_file = self._compile_file
@@ -101,11 +114,40 @@ def compile_lock_path(self):
 
         return output_lock_path(self._compile_file)
 
+    def register_uri(self, uri):
+        self.observed_uri.add(uri)
+
+        file_path = uri2realpath(uri)
+        flags = GetFlags(file_path, self.compile_file, store=self.store)
+
+        if not flags and env.new_file:
+            if filekeys := newfileForCompileFile(
+                file_path, self.compile_file, store=self.store
+            ):
+                # add new file success, update options for module files
+                for v in self.observed_uri:
+                    if filekey(uri2filepath(v)) in filekeys:
+                        self.notify_option_changed(v)
+                return
+
+        if not flags and file_path.endswith(".swift"):
+            flags = InferFlagsForSwift(file_path, self.compile_file, store=self.store)
+
+        self._notify_option_changed(uri, self.optionsForFlags(flags))
+
+    def unregister_uri(self, uri):
+        self.observed_uri.remove(uri)
+
     def optionsForFile(self, uri):
-        file_path = uri2filepath(uri)
-        flags = GetFlags(file_path, self.compile_file, store=self.store)[
-            "flags"
-        ]  # type: list
+        file_path = uri2realpath(uri)
+        flags = GetFlags(file_path, self.compile_file, store=self.store)
+        if not flags and file_path.endswith(".swift"):
+            flags = InferFlagsForSwift(file_path, self.compile_file, store=self.store)
+        return self.optionsForFlags(flags)
+
+    def optionsForFlags(self, flags):
+        if flags is None:
+            return None
         try:
             workdir = flags[flags.index("-working-directory") + 1]
         except (IndexError, ValueError):
@@ -116,19 +158,22 @@ def optionsForFile(self, uri):
         }
 
     def notify_option_changed(self, uri):
-        try:
-            notification = {
-                "jsonrpc": "2.0",
-                "method": "build/sourceKitOptionsChanged",
-                "params": {
-                    "uri": uri,
-                    "updatedOptions": self.optionsForFile(uri),
-                },
-            }
-            send(notification)
-            return True
-        except ValueError as e:  # may have other type change register, like target
-            logger.debug(e)
+        # no clear options?
+        self._notify_option_changed(uri, self.optionsForFile(uri))
+
+    def _notify_option_changed(self, uri, options):
+        # empty options is nouse and lsp will stop working.., at least there should has a infer flags..
+        if options is None:
+            return
+        notification = {
+            "jsonrpc": "2.0",
+            "method": "build/sourceKitOptionsChanged",
+            "params": {
+                "uri": uri,
+                "updatedOptions": options,
+            },
+        }
+        send(notification)
 
     def shutdown(self):
         self.observed_thread = None  # release to end in subthread
@@ -241,6 +286,7 @@ def update_check_time(path):
     def trigger_parse(self, xcpath):
         # FIXME: ensure index_store_path from buildServer.json consistent with parsed .compile file..
         import xclog_parser
+
         xclog_parser.hooks_echo_to_log = True
 
         from xclog_parser import parse, OutputLockedError
@@ -397,10 +443,9 @@ def textDocument_registerForChanges(message):
         action = message["params"]["action"]
         uri = message["params"]["uri"]
         if action == "register":
-            if shared_state.notify_option_changed(uri):
-                shared_state.observed_uri.add(uri)
+            shared_state.register_uri(uri)
         elif action == "unregister":
-            shared_state.observed_uri.remove(uri)
+            shared_state.unregister_uri(uri)
 
     def textDocument_sourceKitOptions(message):
         return {
@@ -438,14 +483,9 @@ def serve():
         message = json.loads(raw)
         logger.debug("Req --> " + raw)
 
-        with lock:
-            response = None
-            handler = dispatch.get(message["method"].replace("/", "_"))
-            if handler:
-                response = handler(message)
-            # ignore other notifications
-            elif "id" in message:
-                response = {
+        def default_response():
+            if "id" in message:
+                return {
                     "jsonrpc": "2.0",
                     "id": message["id"],
                     "error": {
@@ -454,5 +494,17 @@ def serve():
                     },
                 }
 
+        with lock:
+            response = None
+            handler = dispatch.get(message["method"].replace("/", "_"))
+            if handler:
+                try:
+                    response = handler(message)
+                except Exception as e:
+                    logger.exception(f"handle message error: {e}")
+                    response = default_response()
+            else:
+                # ignore other notifications
+                response = default_response()
             if response:
                 send(response)

xcode-build-server

@@ -16,7 +16,7 @@ def show_help():
 
           SPECIAL ENVIRONMENT VARIABLE:
             SOURCEKIT_LOGGING=3: enable detail debug log
-            XBS_LOGPATH: set log path. default is :stderr. use :null to disable log
+            XBS_LOGPATH: set log path. default is `:stderr`. use `:null` to disable log
           """
     )
     exit(0)