Merge pull request #64 from jakedouglas/non_english_anchors

Support non-English characters in anchor names

Author: jch

lib/html/pipeline/toc_filter.rb

@@ -11,9 +11,8 @@ def call
         headers = Hash.new(0)
         doc.css('h1, h2, h3, h4, h5, h6').each do |node|
           name = node.text.downcase
-          name.gsub!(/[^\w\- ]/, '') # remove punctuation
+          name.gsub!(/[^\p{Word}\- ]/u, '') # remove punctuation
           name.gsub!(' ', '-') # replace spaces with dash
-          name = EscapeUtils.escape_uri(name) # escape extended UTF-8 chars
 
           uniq = (headers[name] > 0) ? "-#{headers[name]}" : ''
           headers[name] += 1
@@ -25,4 +24,4 @@ def call
       end
     end
   end
-end
+end

test/html/pipeline/toc_filter_test.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "test_helper"
 
 class HTML::Pipeline::TableOfContentsFilterTest < Test::Unit::TestCase
@@ -42,6 +43,16 @@ def test_all_header_tags_are_found_when_adding_anchors
     doc = TocFilter.call(orig)
     assert_equal 6, doc.search('a').size
   end
-end
 
+  def test_anchors_with_utf8_characters
+    orig = %(<h1>日本語</h1>
+             <h1>Русский</h1)
+
+    rendered_h1s = TocFilter.call(orig).search('h1').map(&:to_s)
 
+    assert_equal "<h1>\n<a name=\"%E6%97%A5%E6%9C%AC%E8%AA%9E\" class=\"anchor\" href=\"#%E6%97%A5%E6%9C%AC%E8%AA%9E\"><span class=\"octicon octicon-link\"></span></a>日本語</h1>",
+                 rendered_h1s[0]
+    assert_equal "<h1>\n<a name=\"%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9\" class=\"anchor\" href=\"#%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9\"><span class=\"octicon octicon-link\"></span></a>Русский</h1>",
+                 rendered_h1s[1]
+  end
+end