Repairing setext heading support, certain uses of backticks after GitHub API format changes (#145)

* Fix parsing of HTML returned from raw API

Github must have changed something on their end, the HTML seems to have changed a little.

* Adding some tests for setext header support

* Fixing the formatter

* Putting all the tests in the same directory

* Closer, but span tag doesn't work in some cases

* Fixed span issue

* Broke test formatting

* fixing local tests - remote is busted, formatting seems very different.

---------

Co-authored-by: Jon Burgess <[email protected]>

Author: coreysciuto-toast

gh-md-toc

@@ -218,6 +218,11 @@ gh_toc(){
 # $2 - number of spaces used to indent.
 #
 gh_toc_grab() {
+
+    href_regex="/href=\"[^\"]+?\"/"
+    lefttext_regex="/(<a class=\"heading-link\".*?>)(.*)(<span aria-hidden)/"
+    righttext_regex="/>.*/"
+
     common_awk_script='
                      modified_href = ""
                      split(href, chars, "")
@@ -242,21 +247,22 @@ gh_toc_grab() {
         echoargs=""
         awkscript='{
                      level = substr($0, 3, 1)
-                     text = substr($0, match($0, />[^<]*<span aria-hidden/)+1, RLENGTH-18)
-                     href = substr($0, match($0, "href=\"([^\"]+)?\"")+6, RLENGTH-7)
+                     lefttext = substr($0, match($0, '$lefttext_regex')+1, RLENGTH-18)
+                     text = substr(lefttext, match(lefttext, '$righttext_regex')+1)
+                     href = substr($0, match($0, '$href_regex')+6, RLENGTH-7)
                      '"$common_awk_script"'
                 }'
     else
         grepcmd="grep -Eo"
         echoargs="-e"
         awkscript='{
                      level = substr($0, 3, 1)
-                     text = substr($0, match($0, />[^<]*<span aria-hidden/)+1, RLENGTH-18)
-                     href = substr($0, match($0, "href=\"[^\"]+?\"")+6, RLENGTH-7)
+                     lefttext = substr($0, match($0, '$lefttext_regex')+1, RLENGTH-18)
+                     text = substr(lefttext, match(lefttext, '$righttext_regex')+1)
+                     href = substr($0, match($0, '$href_regex')+6, RLENGTH-7)
                      '"$common_awk_script"'
                 }'
     fi
-    href_regex='href=\"[^\"]+?\"'
 
     # if closed <h[1-6]> is on the new line, then move it on the prev line
     # for example:
@@ -265,6 +271,9 @@ gh_toc_grab() {
     #   became: The command <code>foo1</code></h1>
     sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' |
 
+    # Sometimes a line can start with <span>. Fix that.
+    sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<span/<span/g' |
+
     # find strings that corresponds to template
     $grepcmd '<h.*id="user-content-[^"]*".*</h[1-6]' |
 

tests/test_backquote.md tests/test directory/test_backquote.mdtests/test_backquote.md renamed to tests/test directory/test_backquote.md

@@ -1,13 +1,8 @@
 # Title
 
 <!--ts-->
-
-- [Title](#title)
-  - [This is test for file path with space](#this-is-test-for-file-path-with-space)
-
-<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
-<!-- Added by: sungminyou, at: 2022년 7월 17일 일요일 16시 48분 18초 KST -->
-
+* [Title](#title)
+   * [This is test for file path with space](#this-is-test-for-file-path-with-space)
 <!--te-->
 
 Blabla...

tests/test directory/test_filepathwithspace.md

@@ -0,0 +1,44 @@
+Title one
+=========
+
+<!--ts-->
+* [Title one](#title-one)
+   * [This is test for setext-style without formatting](#this-is-test-for-setext-style-without-formatting)
+* [<em>Title two</em>](#title-two)
+   * [This is test for setext-style with formatting](#this-is-test-for-setext-style-with-formatting)
+* [Title three](#title-three)
+   * [This is a regression test for atx-style](#this-is-a-regression-test-for-atx-style)
+* [Title four is a particularly long title because of wrapping](#title-four-is-a-particularly-long-title-because-of-wrapping)
+   * [This is a test for long titles](#this-is-a-test-for-long-titles)
+<!--te-->
+
+Blabla...
+
+## This is test for setext-style without formatting
+
+Blabla...
+
+*Title `two`*
+=============
+
+Blabla...
+
+## This is test for setext-style with formatting
+
+Blabla...
+
+# Title `three`
+
+Blabla...
+
+## This is a regression test for atx-style
+
+Blabla...
+
+# Title four is a particularly long title because of wrapping
+
+Blabla...
+
+## This is a test for long titles
+
+Blabla...

tests/test directory/test_nonenglishchars.md

@@ -154,6 +154,16 @@ test_help() {
 }
 
 @test "TOC for non-english chars, #6, #10" {
+    run $BATS_TEST_DIRNAME/../gh-md-toc tests/test\ directory/test_nonenglishchars.md
+    assert_success
+
+    assert_equal "${lines[2]}"   "* [命令行的艺术](#命令行的艺术)"
+    assert_equal "${lines[3]}"   "   * [必读](#必读)"
+    assert_equal "${lines[4]}"   "   * [基础](#基础)"
+    assert_equal "${lines[5]}"   "   * [日常使用](#日常使用)"
+}
+
+@test "TOC for non-english chars (remote load), #6, #10" {
     run $BATS_TEST_DIRNAME/../gh-md-toc \
         https://github.com/ekalinin/envirius/blob/f939d3b6882bfb6ecb28ef7b6e62862f934ba945/README.ru.md
     assert_success
@@ -185,7 +195,7 @@ test_help() {
 }
 
 @test "TOC for text with backquote, #13" {
-    run $BATS_TEST_DIRNAME/../gh-md-toc tests/test_backquote.md
+    run $BATS_TEST_DIRNAME/../gh-md-toc tests/test\ directory/test_backquote.md
     assert_success
 
     assert_equal "${lines[2]}"   "* [The command foo1](#the-command-foo1)"
@@ -196,16 +206,30 @@ test_help() {
 }
 
 @test "TOC for text with plus signs, #100" {
-    run $BATS_TEST_DIRNAME/../gh-md-toc tests/test_plussign.md
+    run $BATS_TEST_DIRNAME/../gh-md-toc tests/test\ directory/test_plussign.md
     assert_success
 
     assert_equal "${lines[2]}" "* [C vs C++](#c-vs-c)"
 }
 
 @test "Toc for file path with space, #136" {
-    run $BATS_TEST_DIRNAME/../gh-md-toc --insert tests/test\ directory/test_filepathwithspace.md
+    run $BATS_TEST_DIRNAME/../gh-md-toc --no-backup --hide-footer tests/test\ directory/test_filepathwithspace.md
     assert_success
 
     assert_equal "${lines[2]}"   "* [Title](#title)"
     assert_equal "${lines[3]}"   "   * [This is test for file path with space](#this-is-test-for-file-path-with-space)"
+}
+
+@test "Toc for setext heading with formatting, #145" {
+    run $BATS_TEST_DIRNAME/../gh-md-toc --no-backup --hide-footer tests/test\ directory/test_setextwithformatting.md
+    assert_success
+
+    assert_equal "${lines[2]}"   "* [Title one](#title-one)"
+    assert_equal "${lines[3]}"   "   * [This is test for setext-style without formatting](#this-is-test-for-setext-style-without-formatting)"
+    assert_equal "${lines[4]}"   "* [<em>Title two</em>](#title-two)"
+    assert_equal "${lines[5]}"   "   * [This is test for setext-style with formatting](#this-is-test-for-setext-style-with-formatting)"
+    assert_equal "${lines[6]}"   "* [Title three](#title-three)"
+    assert_equal "${lines[7]}"   "   * [This is a regression test for atx-style](#this-is-a-regression-test-for-atx-style)"
+    assert_equal "${lines[8]}"   "* [Title four is a particularly long title because of wrapping](#title-four-is-a-particularly-long-title-because-of-wrapping)"
+    assert_equal "${lines[9]}"   "   * [This is a test for long titles](#this-is-a-test-for-long-titles)"
 }