Merge pull request #1 from mbeijen/claude/fix-encoding

Fix UTF-8 encoding in scraper; add DB repair script

Author: mbeijen

lib/tasks/get_latest.rake

@@ -15,7 +15,7 @@
   	  	gameIds = Array.new
 	  	for i in args.arg1.to_i..args.arg2.to_i
 	  		seasonsUrl = 'http://j-archive.com/showseason.php?season='+i.to_s
-	  		seasonList = Nokogiri::HTML(URI.open(seasonsUrl))
+	  		seasonList = Nokogiri::HTML(URI.open(seasonsUrl).read, nil, 'UTF-8')
 	  		linkList = seasonList.css('table td a')
 	  		linkList.each do |ll|
 	  			href = ll.attr('href');
@@ -27,7 +27,7 @@
 
 	  	gameIds.each do |gid|
 		  	gameurl = 'http://www.j-archive.com/showgame.php?game_id='+gid.to_s
-		  	game = Nokogiri::HTML(URI.open(gameurl))
+		  	game = Nokogiri::HTML(URI.open(gameurl).read, nil, 'UTF-8')
 
 		  	## OK, were going to do this twice, once for each round
 		  	questions = game.css("#jeopardy_round .clue")
@@ -66,8 +66,7 @@
 					answerDiv = game.css("#".concat(id))
 
 					#=========== Set Answer =============
-		  			answermatch = /ponse">(.*)<\/e/.match(answerDiv.to_html)
-                                        var_answer = answermatch.captures[0].to_s
+		  			var_answer = answerDiv.css('em.correct_response').first&.text().to_s
                                         var_question = q.css('.clue_text').first.text()
 					index =	q.xpath('count(preceding-sibling::*)').to_i
 					var_category = categoryArr[index]

script/fix_encoding.rb

@@ -0,0 +1,49 @@
+# Repair mojibake in clues imported before the Nokogiri UTF-8 fix.
+#
+# Root cause: Nokogiri parsed j-archive.com (UTF-8) as Latin-1, so each
+# multi-byte UTF-8 sequence was stored as individual Latin-1 codepoints.
+# Questions have plain mojibake ("Elysée"). Answers additionally have
+# HTML-entity-encoded mojibake ("apéritif") because the
+# scraper used .to_html before extracting the text.
+#
+# Fix: for questions, re-encode as ISO-8859-1 bytes then force UTF-8.
+#      for answers, HTML-unescape first, then same re-encoding.
+#
+# Run with:
+#   cd /srv/jservice && rails runner script/fix_encoding.rb
+#
+require 'cgi'
+
+def fix_mojibake(s)
+  s.encode('ISO-8859-1').force_encoding('UTF-8')
+rescue Encoding::UndefinedConversionError, EncodingError
+  s
+end
+
+fixed_q = 0
+fixed_a = 0
+errors  = 0
+
+Clue.find_each do |clue|
+  updates = {}
+
+  q_fixed = fix_mojibake(clue.question)
+  if q_fixed != clue.question && q_fixed.valid_encoding?
+    updates[:question] = q_fixed
+    fixed_q += 1
+  end
+
+  a_unescaped = CGI.unescapeHTML(clue.answer)
+  a_fixed = fix_mojibake(a_unescaped)
+  if a_fixed != clue.answer && a_fixed.valid_encoding?
+    updates[:answer] = a_fixed
+    fixed_a += 1
+  end
+
+  clue.update_columns(updates) unless updates.empty?
+rescue => e
+  errors += 1
+  warn "Clue #{clue.id}: #{e}"
+end
+
+puts "Done. Fixed #{fixed_q} questions and #{fixed_a} answers (#{errors} errors)."