Fixed further locations where output encoding was not preserved. Bug #11 by Vít Ondruch

Author: drbrain

History.txt

@@ -1,15 +1,17 @@
-=== 3.4.1
+=== 3.5
 
 * Minor enhancements
   * RDoc::Parser::C looks for rb_scan_args and fills in RDoc::AnyMethod#params
     appropriately.  This may provide useful information if the author did not
     provide a call-seq.
   * RDoc::Parser::C now records the function name for methods implemented in
     C.
-  * RDoc now records file and byte offset inforamtion for methods.
+  * RDoc now records file and byte offset information for methods.
 * Bug fixes
   * Locations of module aliases are now recorded.
   * RDoc::Parser::C finds method bodies better now.
+  * Fixed further locations where output encoding was not preserved.  Bug #11
+    by Vít Ondruch.
 
 === 3.4
 

lib/rdoc.rb

@@ -95,7 +95,7 @@ def self.const_missing const_name # :nodoc:
   ##
   # RDoc version you are using
 
-  VERSION = '3.4.1'
+  VERSION = '3.5'
 
   ##
   # Method visibilities

lib/rdoc/parser/ruby.rb

@@ -185,6 +185,7 @@ def initialize(top_level, file_name, content, options, stats)
   def collect_first_comment
     skip_tkspace
     comment = ''
+    comment.force_encoding @encoding if @encoding
     first_line = true
 
     tk = get_tk
@@ -1259,6 +1260,8 @@ def parse_require(context, comment)
 
   def parse_statements(container, single = NORMAL, current_method = nil,
                        comment = '')
+    comment.force_encoding @encoding if @encoding
+
     nest = 1
     save_visibility = container.visibility
 
@@ -1282,6 +1285,7 @@ def parse_statements(container, single = NORMAL, current_method = nil,
               comment.empty?
 
             comment = ''
+            comment.force_encoding @encoding if @encoding
           end
 
           while TkCOMMENT === tk do
@@ -1421,7 +1425,10 @@ def parse_statements(container, single = NORMAL, current_method = nil,
         keep_comment = false
       end
 
-      comment = '' unless keep_comment
+      unless keep_comment then
+        comment = ''
+        comment.force_encoding @encoding if @encoding
+      end
 
       begin
         get_tkread

lib/rdoc/text.rb

@@ -97,7 +97,8 @@ def normalize_comment text
     text = strip_hashes text
     text = expand_tabs text
     text = flush_left text
-    strip_newlines text
+    text = strip_newlines text
+    text
   end
 
   ##
@@ -139,7 +140,11 @@ def parse text
 
   def strip_hashes text
     return text if text =~ /^(?>\s*)[^\#]/
-    text.gsub(/^\s*(#+)/) { $1.tr '#',' ' }.gsub(/^\s+$/, '')
+
+    empty = ''
+    empty.force_encoding text.encoding if Object.const_defined? :Encoding
+
+    text.gsub(/^\s*(#+)/) { $1.tr '#', ' ' }.gsub(/^\s+$/, empty)
   end
 
   ##

test/test_rdoc_parser_ruby.rb

@@ -32,6 +32,36 @@ def teardown
     @tempfile2.close
   end
 
+  def test_collect_first_comment
+    p = util_parser <<-CONTENT
+# first
+
+# second
+class C; end
+    CONTENT
+
+    comment = p.collect_first_comment
+
+    assert_equal "# first\n", comment
+  end
+
+  def test_collect_first_comment_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    @options.encoding = Encoding::CP852
+
+    p = util_parser <<-CONTENT
+# first
+
+# second
+class C; end
+    CONTENT
+
+    comment = p.collect_first_comment
+
+    assert_equal Encoding::CP852, comment.encoding
+  end
+
   def test_extract_call_seq
     m = RDoc::AnyMethod.new nil, 'm'
     p = util_parser ''
@@ -1439,6 +1469,28 @@ def test_parse_statements_class_nested
     assert_equal 'my method', bar.comment
   end
 
+  def test_parse_statements_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+    @options.encoding = Encoding::CP852
+
+    content = <<-EOF
+class Foo
+  ##
+  # this is my method
+  add_my_method :foo
+end
+    EOF
+
+    util_parser content
+
+    @parser.parse_statements @top_level
+
+    foo = @top_level.classes.first.method_list.first
+    assert_equal 'foo', foo.name
+    assert_equal 'this is my method', foo.comment
+    assert_equal Encoding::CP852, foo.comment.encoding
+  end
+
   def test_parse_statements_identifier_meta_method
     content = <<-EOF
 class Foo
@@ -1450,7 +1502,7 @@ class Foo
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.method_list.first
     assert_equal 'foo', foo.name
@@ -1466,7 +1518,7 @@ def foo() end
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.method_list[0]
     assert_equal 'foo', foo.name
@@ -1499,7 +1551,7 @@ def foo()
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.method_list[0]
     assert_equal 'foo', foo.name
@@ -1565,7 +1617,7 @@ class Foo
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     constants = @top_level.classes.first.constants
 
@@ -1612,7 +1664,7 @@ def test_parse_statements_identifier_attr
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.attributes.first
     assert_equal 'foo', foo.name
@@ -1624,7 +1676,7 @@ def test_parse_statements_identifier_attr_accessor
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.attributes.first
     assert_equal 'foo', foo.name
@@ -1636,7 +1688,7 @@ def test_parse_statements_identifier_include
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first
     assert_equal 'Foo', foo.name
@@ -1648,7 +1700,7 @@ def test_parse_statements_identifier_module_function
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo, s_foo = @top_level.modules.first.method_list
     assert_equal 'foo',    foo.name,       'instance method name'
@@ -1665,7 +1717,7 @@ def test_parse_statements_identifier_private
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.classes.first.method_list.first
     assert_equal 'foo', foo.name
@@ -1677,7 +1729,7 @@ def test_parse_statements_identifier_require
 
     util_parser content
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     assert_equal 1, @top_level.requires.length
   end
@@ -1695,7 +1747,7 @@ def b
 end
     RUBY
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     c_a = @top_level.classes.first
     assert_equal 'A', c_a.full_name
@@ -1887,7 +1939,7 @@ class Baz
       end
     EOS
 
-    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil, ''
+    @parser.parse_statements @top_level
 
     foo = @top_level.modules.first.modules.first
     assert_equal 'Foo', foo.name

test/test_rdoc_text.rb

@@ -134,6 +134,31 @@ def test_strip_hashes
     assert_equal expected, strip_hashes(text)
   end
 
+  def test_strip_hashes_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    text = <<-TEXT
+##
+# we don't worry too much.
+#
+# The comments associated with
+    TEXT
+
+    text.force_encoding Encoding::CP852
+
+    expected = <<-EXPECTED
+
+  we don't worry too much.
+
+  The comments associated with
+    EXPECTED
+
+    stripped = strip_hashes text
+
+    assert_equal expected, stripped
+    assert_equal Encoding::CP852, stripped.encoding
+  end
+
   def test_strip_newlines
     assert_equal ' ',  strip_newlines("\n \n")
 
@@ -178,6 +203,54 @@ def test_strip_stars
     assert_equal expected, strip_stars(text)
   end
 
+  def test_strip_stars_encoding
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+
+    text = <<-TEXT
+/*
+ * * we don't worry too much.
+ *
+ * The comments associated with
+ */
+    TEXT
+
+    text.force_encoding Encoding::CP852
+
+    expected = <<-EXPECTED
+
+   * we don't worry too much.
+
+   The comments associated with
+    EXPECTED
+
+    assert_equal expected, strip_stars(text)
+    assert_equal Encoding::CP852, text.encoding
+  end
+
+  def test_strip_hashes_encoding
+
+    text = <<-TEXT
+##
+# we don't worry too much.
+#
+# The comments associated with
+    TEXT
+
+    text.force_encoding Encoding::CP852
+
+    expected = <<-EXPECTED
+
+  we don't worry too much.
+
+  The comments associated with
+    EXPECTED
+
+    stripped = strip_hashes text
+
+    assert_equal expected, stripped
+    assert_equal Encoding::CP852, stripped.encoding
+  end
+
   def test_to_html_apostrophe
     assert_equal '‘a', to_html("'a")
     assert_equal 'a’', to_html("a'")