Merge pull request #1192 from Kraymer/lyrics-fix-incomplete

lyrics : remove empty divs before scraping

Author: Kraymer

beetsplug/lyrics.py

@@ -90,7 +90,7 @@ def extract_text_between(html, start_marker, end_marker):
         html, _ = html.split(end_marker, 1)
     except ValueError:
         return u''
-    return _scrape_strip_cruft(html, True)
+    return html
 
 
 def extract_text_in(html, starttag):
@@ -124,8 +124,7 @@ def extract_text_in(html, starttag):
     else:
         print('no closing tag found!')
         return
-    lyrics = ''.join(parts)
-    return _scrape_strip_cruft(lyrics, True)
+    return u''.join(parts)
 
 
 def search_pairs(item):
@@ -221,7 +220,7 @@ def fetch_lyricswiki(artist, title):
     if not html:
         return
 
-    lyrics = extract_text_in(html, "<div class='lyricbox'>")
+    lyrics = extract_text_in(html, u"<div class='lyricbox'>")
     if lyrics and 'Unfortunately, we are not licensed' not in lyrics:
         return lyrics
 
@@ -360,13 +359,14 @@ def _scrape_strip_cruft(html, plain_text_out=False):
         html = COMMENT_RE.sub('', html)
         html = TAG_RE.sub('', html)
 
-    # Strip lines
     html = '\n'.join([x.strip() for x in html.strip().split('\n')])
+    html = re.sub(r'\n{3,}', r'\n\n', html)
     return html
 
 
 def _scrape_merge_paragraphs(html):
-    return re.sub(r'</p>\s*<p(\s*[^>]*)>', '\n', html)
+    html = re.sub(r'</p>\s*<p(\s*[^>]*)>', '\n', html)
+    return re.sub(r'<div .*>\s*</div>', '\n', html)
 
 
 def scrape_lyrics_from_html(html):
@@ -541,4 +541,4 @@ def get_lyrics(self, artist, title):
             if lyrics:
                 log.debug(u'got lyrics from backend: {0}'
                           .format(backend.__name__))
-                return lyrics.strip()
+                return _scrape_strip_cruft(lyrics, True)

docs/changelog.rst

@@ -30,6 +30,8 @@ New:
 
 Fixed:
 
+* :doc:`/plugins/lyrics`: Avoid fetching truncated lyrics from the Google
+  backed by merging text blocks separated by empty ``<div>`` before scraping.
 * Fix a new crash with the latest version of Mutagen (1.26).
 * We now print a better error message when the database file is corrupted.
 * :doc:`/plugins/discogs`: Only prompt for authentication when running the

test/rsrc/lyrics/examplecom/beetssong.txt

@@ -222,7 +222,9 @@ e9.size = "120x600, 160x600";
         <h2>John Doe <br> beets song lyrics</h2>
 <img src="images/phone-left.gif" alt="Ringtones left icon" width="16" height="17"> <a href="http://www.ringtonematcher.com/go/?sid=LBSMros&amp;artist=The+John Doe&amp;song=Beets+Song" target="_blank"><b><font size="+1" color="red" face="arial">Send "beets song" Ringtone to your Cell</font></b></a> <img src="images/phone-right.gif" alt="Ringtones right icon" width="16" height="17"><br><br><center>Beets is the media library management system for obsessive-compulsive music geeks.<br>
 The purpose of beets is to get your music collection right once and for all. It catalogs your collection, automatically improving its metadata as it goes. It then provides a bouquet of tools for manipulating and accessing your music.<br>
-Here's an example of beets' brainy tag corrector doing its thing:</center>
+<div class='flow breaker'>           </div>
+Here's an example of beets' brainy tag corrector doing its thing:
+Because beets is designed as a library, it can do almost anything you can imagine for your music collection. Via plugins, beets becomes a panacea</center>
 <img src="images/phone-left.gif" alt="Ringtones left icon" width="16" height="17"> <a href="http://www.ringtonematcher.com/go/?sid=LBSMros&amp;artist=The+John Doe&amp;song=Beets+Song" target="_blank"><b><font size="+1" color="red" face="arial">Send "beets song" Ringtone to your Cell</font></b></a> <img src="images/phone-right.gif" alt="Ringtones right icon" width="16" height="17"><br><br>
         <center>
          <font color="black" size="2" face="arial">Share <strong>beets song lyrics</strong></font><br><p style="height: 1px; margin: 3pt; padding: 0pt;"></p>

test/rsrc/lyricstext.yaml

@@ -1,3 +1,8 @@
+Beets_song:
+    - geeks
+    - bouquet
+    - panacea
+
 Amsterdam:
     - oriflammes
     - fortune

test/test_lyrics.py

@@ -300,6 +300,15 @@ def setUp(self):
         lyrics.LyricsPlugin()
         lyrics.fetch_url = MockFetchUrl()
 
+    def test_mocked_source_ok(self):
+        """Test that lyrics of the mocked page are correctly scraped"""
+        url = self.source['url'] + self.source['path']
+        if os.path.isfile(url_to_filename(url)):
+            res = lyrics.scrape_lyrics_from_html(lyrics.fetch_url(url))
+            self.assertTrue(lyrics.is_lyrics(res), url)
+            self.assertTrue(is_lyrics_content_ok(self.source['title'], res),
+                            url)
+
     def test_google_sources_ok(self):
         """Test if lyrics present on websites registered in beets google custom
         search engine are correctly scraped."""
@@ -319,12 +328,11 @@ def test_default_ok(self):
         for (fun, s) in zip([lyrics.fetch_lyricswiki,
                              lyrics.fetch_lyricscom,
                              lyrics.fetch_musixmatch], DEFAULT_SOURCES):
-            if os.path.isfile(url_to_filename(
-                              s['url'] + s['path'])):
+            url = s['url'] + s['path']
+            if os.path.isfile(url_to_filename(url)):
                 res = fun(s['artist'], s['title'])
-                self.assertTrue(lyrics.is_lyrics(res))
-                self.assertTrue(is_lyrics_content_ok(
-                                s['title'], res))
+                self.assertTrue(lyrics.is_lyrics(res), url)
+                self.assertTrue(is_lyrics_content_ok(s['title'], res), url)
 
     def test_is_page_candidate_exact_match(self):
         """Test matching html page title with song infos -- when song infos are