Version bump v2.0.0-beta.20

+ Using main thread for most things and sending thread for sending commands
+ Stream for error handler always using "sys.stderr"
+ ECMASCript: Fix extra paths name

Author: Kronuz

codeintel/__init__.py

@@ -1 +1 @@
-__version__ = '2.0.0-beta.19'
+__version__ = '2.0.0-beta.20'

codeintel/__main__.py

@@ -55,8 +55,14 @@ def flush(self):
 
 
 class BinaryStream(object):
-    def __init__(self, stream):
-        self.stream = stream
+    def __init__(self, stream=None):
+        self._stream = stream
+
+    @property
+    def stream(self):
+        if self._stream is not None:
+            return self._stream
+        return sys.stderr
 
     def encode(self, message):
         if isinstance(message, six.text_type):
@@ -94,7 +100,7 @@ class Shell(cmdln.Cmdln):
     def __init__(self, *args, **kwargs):
         cmdln.Cmdln.__init__(self, *args, **kwargs)
 
-        logging.basicConfig(stream=TextStream(sys.stderr))
+        logging.basicConfig(stream=TextStream())
 
         # Don't redirect output
         os.environ["KOMODO_VERBOSE"] = "1"
@@ -197,19 +203,14 @@ def set_verbosity(self, option, opt_str, value, parser):
             logging.getLogger('codeintel').setLevel(logging.DEBUG)
 
     def set_log_file(self, option, opt_str, value, parser):
-        if value in ('stdout', '/dev/stdout'):
-            stream = sys.stdout
-        elif value in ('stderr', '/dev/stderr'):
-            stream = sys.stderr
-        else:
+        if value not in ('stdout', '/dev/stdout', 'stderr', '/dev/stderr'):
             log_dir = os.path.dirname(value)
             if not os.path.exists(log_dir):
                 os.makedirs(log_dir)
             stream = open(value, 'wt')
-        logging.getLogger().addHandler(logging.StreamHandler(stream=TextStream(stream)))
-        # XXX marky horrible ugly hack
-        sys.stderr = stream
-        sys.stdout = stream
+            # XXX marky horrible ugly hack
+            sys.stderr = stream
+            sys.stdout = stream
 
     def get_optparser(self):
         optparser = cmdln.Cmdln.get_optparser(self)

codeintel/client.py

@@ -51,12 +51,19 @@
 PRIORITY_BACKGROUND = 4     # info may be needed sometime
 
 logger_name = 'CodeIntel.codeintel'
+logger_level = logging.INFO  # INFO
 
-logging.getLogger(logger_name).setLevel(logging.INFO)  # INFO
+logger = logging.getLogger(logger_name)
+logger.setLevel(logger_level)
 
 
 class CodeIntel(object):
-    def __init__(self):
+    def __init__(self, main_thread_runner):
+        """
+        main_thread_runner - Must be a function receiving a single parameter,
+                                a function to be executed in the main thread.
+        """
+        self._main_thread_runner = main_thread_runner
         self.log = logging.getLogger(logger_name + '.' + self.__class__.__name__)
         self.mgr = None
         self._mgr_lock = threading.Lock()
@@ -151,7 +158,7 @@ def activate(self, reset_db_as_necessary=False, codeintel_command=None, oop_mode
 
     @property
     def enabled(self):
-        return self._enabled
+        return self._enabled and self.mgr and self.mgr.is_alive()
 
     def deactivate(self):
         with self._mgr_lock:
@@ -449,6 +456,7 @@ def __init__(self, service, progress_callback=None, shutdown_callback=None, code
         if env is not None:
             self.env = env
         self._state_condvar = threading.Condition()
+        self._discard_time = time.time()
         self.requests = {}  # keyed by request id; value is tuple (callback, request data, time sent) requests will time out at some point...
         self.unsent_requests = queue.Queue()
         threading.Thread.__init__(self, name="CodeIntel Manager Thread")
@@ -469,26 +477,29 @@ def start(self, reset_db_as_necessary=False):
 
     def shutdown(self):
         """Abort any outstanding requests and shut down gracefully"""
-        self.abort()
         if self.state is CodeIntelManager.STATE_DESTROYED:
             return  # already dead
+        self.abort()
+        self.quit()
         if not self.pipe:
             # not quite dead, but already disconnected... ungraceful shutdown
             self.kill()
             return
-        self._send(command='quit', callback=self.do_quit)
-        self.state = CodeIntelManager.STATE_QUITTING
 
     def abort(self):
         """Abort all running requests"""
         for req in list(self.requests.keys()):
             self._abort.add(req)
-            self._send(
+            self.send(
                 command='abort',
                 id=req,
                 callback=lambda request, response: None,
             )
 
+    def quit(self):
+        self.send(command='quit', callback=self.do_quit)
+        self.state = CodeIntelManager.STATE_QUITTING
+
     def close(self):
         try:
             self.pipe.close()
@@ -739,10 +750,10 @@ def initialization_completed():
                 self._send_request_thread.start()
             update("CodeIntel ready.", state=CodeIntelManager.STATE_READY)
 
-        self._send(callback=get_cpln_langs, command='get-languages', type='cpln')
         self._send(callback=get_citadel_langs, command='get-languages', type='citadel')
         self._send(callback=get_xml_langs, command='get-languages', type='xml')
         self._send(callback=get_stdlib_langs, command='get-languages', type='stdlib-supported')
+        self._send(callback=get_cpln_langs, command='get-languages', type='cpln')
 
         self.set_global_environment(self.env, self.prefs)
 
@@ -756,7 +767,7 @@ def update_callback(response):
     def set_global_environment(self, env, prefs):
         self.env = env
         self.prefs = [prefs] if isinstance(prefs, dict) else prefs
-        self._send(
+        self.send(
             command='set-environment',
             env=self.env,
             prefs=self.prefs,
@@ -768,7 +779,7 @@ def get_available_catalogs(request, response):
                 self.available_catalogs = response.get('catalogs', [])
             if update_callback:
                 update_callback(response)
-        self._send(callback=get_available_catalogs, command='get-available-catalogs')
+        self.send(callback=get_available_catalogs, command='get-available-catalogs')
 
     def send(self, callback=None, **kwargs):
         """Public API for sending a request.
@@ -783,11 +794,12 @@ def send(self, callback=None, **kwargs):
 
     def _send_queued_requests(self):
         """Worker to send unsent requests"""
-        while True:
-            with self._state_condvar:
-                if self.state is CodeIntelManager.STATE_DESTROYED:
-                    break  # Manager already shut down
-                if self.state is not CodeIntelManager.STATE_READY:
+
+        self.log.info("%s thread started..." % threading.current_thread().name)
+
+        while self.state not in (CodeIntelManager.STATE_QUITTING, CodeIntelManager.STATE_DESTROYED):
+            if self.state is not CodeIntelManager.STATE_READY:
+                with self._state_condvar:
                     self._state_condvar.wait()
                     continue  # wait...
             callback, kwargs = self.unsent_requests.get()
@@ -796,6 +808,8 @@ def _send_queued_requests(self):
                 break
             self._send(callback, **kwargs)
 
+        self.log.info("%s thread ended!" % threading.current_thread().name)
+
     def _send(self, callback=None, **kwargs):
         """
         Private API for sending; ignores the current state of the manager and
@@ -804,8 +818,6 @@ def _send(self, callback=None, **kwargs):
         calling thread until the data has been written (though possibly not yet
         received on the other end).
         """
-        if not self.pipe or self.state is CodeIntelManager.STATE_QUITTING:
-            return  # Nope, eating all commands during quit
         req_id = hex(self._next_id)
         kwargs['req_id'] = req_id
         text = json.dumps(kwargs, separators=(',', ':'))
@@ -833,17 +845,16 @@ def run(self):
         assert threading.current_thread().name != "MainThread", \
             "CodeIntelManager.run should run on background thread!"
 
-        self.log.debug("CodeIntelManager thread started...")
+        self.log.info("%s thread started..." % threading.current_thread().name)
 
-        while True:
+        while self.state not in (CodeIntelManager.STATE_QUITTING, CodeIntelManager.STATE_DESTROYED):
             ok = False
 
             self.init_child()
             if not self.proc:
                 break  # init child failed
 
             first_buf = True
-            discard_time = 0.0
             try:
                 buf = b''
                 while self.proc and self.pipe:
@@ -875,20 +886,6 @@ def run(self):
                             raise ValueError("Invalid frame length character: %r" % ch)
                         buf += ch
 
-                    now = time.time()
-                    if now - discard_time > 60:  # discard some stale results
-                        for req_id, (callback, request, sent_time) in list(self.requests.items()):
-                            if sent_time < now - 5 * 60:
-                                # sent 5 minutes ago - it's irrelevant now
-                                try:
-                                    if callback:
-                                        callback(request, {})
-                                except Exception as e:
-                                    self.log.error("Failed timing out request")
-                                else:
-                                    self.log.debug("Discarding request %r", request)
-                                del self.requests[req_id]
-
             except Exception as e:
                 if self.state in (CodeIntelManager.STATE_QUITTING, CodeIntelManager.STATE_DESTROYED):
                     self.log.debug("IOError in codeintel during shutdown; ignoring")
@@ -902,43 +899,63 @@ def run(self):
             if not ok:
                 time.sleep(3)
 
-        self.log.debug("CodeIntelManager thread ended!")
+        self.log.info("%s thread ended!" % threading.current_thread().name)
 
     def handle(self, response):
         """Handle a response from the codeintel process"""
-        self.log.debug("handling: %r", response)
-        req_id = response.get('req_id')
-        callback, request, sent_time = self.requests.get(req_id, (None, None, None))
-        request_command = request.get('command', '') if request else None
-        response_command = response.get('command', request_command)
-        if req_id is None or request_command != response_command:
-            # unsolicited response, look for a handler
-            try:
-                if not response_command:
-                    self.log.error("No 'command' in response %r", response)
-                    raise ValueError("Invalid response frame %r" % response)
-                meth = getattr(self, 'do_' + response_command.replace('-', '_'), None)
-                if not meth:
-                    self.log.error("Unknown command %r, response %r", response_command, response)
-                    raise ValueError("Unknown unsolicited response \"%s\"" % response_command)
-                meth(response)
-            except Exception as e:
-                self.log.error("Error handling unsolicited response")
-            return
-        if not request:
-            self.log.error("Discard response for unknown request %s (command %s): have %s",
-                      req_id, response_command or '%r' % response, sorted(self.requests.keys()))
-            return
-        self.log.debug("Request %s (command %s) took %0.2f seconds", req_id, request_command or '<unknown>', time.time() - sent_time)
-        if 'success' in response:
-            # remove completed request
-            self.log.debug("Removing completed request %s", req_id)
-            del self.requests[req_id]
-        else:
-            # unfinished response; update the sent time so it doesn't time out
-            self.requests[req_id] = (callback, request, time.time())
-        if callback:
-            callback(request, response)
+        def _handle():
+            assert threading.current_thread().name == "MainThread", \
+                "CodeIntelManager.handle() should run on main thread!"
+
+            now = time.time()
+            if now - self._discard_time > 60:  # (not so often) discard some stale results
+                self._discard_time = now
+                for req_id, (callback, request, sent_time) in list(self.requests.items()):
+                    if sent_time < now - 5 * 60:
+                        # sent 5 minutes ago - it's irrelevant now
+                        try:
+                            if callback:
+                                callback(request, {})
+                        except Exception as e:
+                            self.log.error("Failed timing out request")
+                        else:
+                            self.log.debug("Discarding request %r", request)
+                        del self.requests[req_id]
+
+            self.log.debug("handling: %r", response)
+            req_id = response.get('req_id')
+            callback, request, sent_time = self.requests.get(req_id, (None, None, None))
+            request_command = request.get('command', '') if request else None
+            response_command = response.get('command', request_command)
+            if req_id is None or request_command != response_command:
+                # unsolicited response, look for a handler
+                try:
+                    if not response_command:
+                        self.log.error("No 'command' in response %r", response)
+                        raise ValueError("Invalid response frame %r" % response)
+                    meth = getattr(self, 'do_' + response_command.replace('-', '_'), None)
+                    if not meth:
+                        self.log.error("Unknown command %r, response %r", response_command, response)
+                        raise ValueError("Unknown unsolicited response \"%s\"" % response_command)
+                    meth(response)
+                except Exception as e:
+                    self.log.error("Error handling unsolicited response")
+                return
+            if not request:
+                self.log.error("Discard response for unknown request %s (command %s): have %s",
+                        req_id, response_command or '%r' % response, sorted(self.requests.keys()))
+                return
+            self.log.debug("Request %s (command %s) took %0.2f seconds", req_id, request_command or '<unknown>', time.time() - sent_time)
+            if 'success' in response:
+                # remove completed request
+                self.log.debug("Removing completed request %s", req_id)
+                del self.requests[req_id]
+            else:
+                # unfinished response; update the sent time so it doesn't time out
+                self.requests[req_id] = (callback, request, time.time())
+            if callback:
+                callback(request, response)
+        self.service._main_thread_runner(_handle)  # Do handling in main thread
 
     def do_scan_complete(self, response):
         """Scan complete unsolicited response"""

codeintel/codeintel2/lang_ecma.py

@@ -218,8 +218,8 @@ class ECMAScriptLangIntel(CitadelLangIntel,
                           PythonCITDLExtractorMixin):
     lang = lang
     interpreterPrefName = "node"
-    extraPathsPrefName = "esExtraPaths"
-    excludePathsPrefName = "esExcludePaths"
+    extraPathsPrefName = "ecmascriptExtraPaths"
+    excludePathsPrefName = "ecmascriptExcludePaths"
 
     # Define the trigger chars we use, used by ProgLangTriggerIntelMixin
     trg_chars = tuple(".(,@'\" ")

codeintel/codeintel2/oop/driver.py

@@ -508,15 +508,15 @@ def start(self):
                 self.quit = True
                 break
             if ch == b'{':
-                size = int(buf, 10)
+                length = int(buf, 10)
                 try:
                     buf = ch
-                    while len(buf) < size:
-                        last_size = len(buf)
-                        buf += self.fd_in.read(size - len(buf))
-                        if len(buf) == last_size:
+                    while len(buf) < length:
+                        data = self.fd_in.read(length - len(buf))
+                        if not data:
                             # nothing read, EOF
                             raise IOError("Failed to read frame from socket")
+                        buf += data
                 except IOError:
                     log.debug("Failed to read frame data, assuming connection died")
                     self.quit = True