stop sending/receiving ids between Ruby and Python (#218)

Author: slonopotamus

CHANGELOG.adoc

@@ -5,6 +5,10 @@
 This document provides a high-level view of the changes to the {project-name} by release.
 For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
 
+== Unreleased
+
+* stop sending/receiving `ids` between Ruby and Python
+
 == 2.0.0.rc3 (2021-01-08) - @slonopotamus
 
 * fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])

lib/pygments/mentos.py

@@ -196,27 +196,6 @@ def _send_data(self, res, method):
         sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -261,18 +240,9 @@ def start(self):
                 # Read up to the given number of *bytes* (not chars) (possibly 0)
                 text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
-
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:

lib/pygments/popen.rb

@@ -237,7 +237,6 @@ def with_watchdog(timeout_time, error_message)
             stop error_message
             state = :timeout
           end
-
         end
       end : nil
       begin
@@ -258,7 +257,7 @@ def with_watchdog(timeout_time, error_message)
 
     # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
     # args, kwargs, code.
-    def mentos(method, args = [], kwargs = {}, original_code = nil)
+    def mentos(method, args = [], kwargs = {}, code = nil)
       # Open the pipe if necessary
       start unless alive?
 
@@ -273,12 +272,6 @@ def mentos(method, args = [], kwargs = {}, original_code = nil)
                        end
       end
 
-      # For sanity checking on both sides of the pipe when highlighting, we prepend and
-      # append an id.  mentos checks that these are 8 character ids and that they match.
-      # It then returns the id's back to Rubyland.
-      id = (0...8).map { rand(65..89).chr }.join
-      code = original_code ? add_ids(original_code, id) : nil
-
       # Add metadata to the header and generate it.
       bytesize = if code
                    code.bytesize
@@ -287,7 +280,7 @@ def mentos(method, args = [], kwargs = {}, original_code = nil)
                  end
 
       kwargs.freeze
-      kwargs = kwargs.merge('fd' => @out.to_i, 'id' => id, 'bytes' => bytesize)
+      kwargs = kwargs.merge('fd' => @out.to_i, 'bytes' => bytesize)
       out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
 
       begin
@@ -315,7 +308,7 @@ def mentos(method, args = [], kwargs = {}, original_code = nil)
           header = @out.read(header_len)
 
           # Now handle the header, any read any more data required.
-          handle_header_and_return(header, id)
+          handle_header_and_return(header)
         end
 
         # Finally, return what we got.
@@ -341,10 +334,8 @@ def mentos(method, args = [], kwargs = {}, original_code = nil)
     # Based on the header we receive, determine if we need
     # to read more bytes, and read those bytes if necessary.
     #
-    # Then, do a sanity check with the ids.
-    #
     # Returns a result - either highlighted text or metadata.
-    def handle_header_and_return(header, id)
+    def handle_header_and_return(header)
       if header
         @log.info "In header: #{header}"
         header = header_to_json(header)
@@ -356,37 +347,14 @@ def handle_header_and_return(header, id)
         if header[:method] == 'highlight'
           # Make sure we have a result back; else consider this an error.
           raise MentosError, 'No highlight result back from mentos.' if res.nil?
-
-          @log.info 'Highlight in process.'
-
-          # Get the id's
-          start_id = res[0..7]
-          end_id = res[-8..-1]
-
-          # Sanity check.
-          if !((start_id == id) && (end_id == id))
-            raise MentosError, "ID's did not match. Aborting."
-          else
-            # We're good. Remove the padding
-            res = res[10..-11]
-            @log.info 'Highlighting complete.'
-            res
-          end
         end
+
         res
       else
         raise MentosError, 'No header received back.'
       end
     end
 
-    # With the code, prepend the id (with two spaces to avoid escaping weirdness if
-    # the following text starts with a slash (like terminal code), and append the
-    # id, with two padding also. This means we are sending over the 8 characters +
-    # code + 8 characters.
-    def add_ids(code, id)
-      (id + "  #{code}  #{id}").freeze
-    end
-
     # Return the final result for the API. Return Ruby objects for the methods that
     # want them, text otherwise.
     def return_result(res, method)

test/test_pygments.rb

@@ -119,31 +119,6 @@ def test_highlight_on_multi_threads
   end
 end
 
-# Philosophically, I'm not the biggest fan of testing private
-# methods, but given the relative delicacy of validity checking
-# over the pipe I think it's necessary and informative.
-class PygmentsValidityTest < Test::Unit::TestCase
-  def test_add_ids_with_padding
-    res = PE.send(:add_ids, 'herp derp baz boo foo', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  herp derp baz boo foo  ABCDEFGH', res
-  end
-
-  def test_add_ids_on_empty_string
-    res = PE.send(:add_ids, '', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH    ABCDEFGH', res
-  end
-
-  def test_add_ids_with_unicode_data
-    res = PE.send(:add_ids, '# ø ø ø', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  # ø ø ø  ABCDEFGH', res
-  end
-
-  def test_add_ids_with_starting_slashes
-    res = PE.send(:add_ids, '\\# ø ø ø..//', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  \\# ø ø ø..//  ABCDEFGH', res
-  end
-end
-
 class PygmentsLexerTest < Test::Unit::TestCase
   RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"