解决字幕库抓取限制问题

Author: Andyfoo

pom.xml

@@ -5,7 +5,7 @@
 
 	<groupId>SubTitleSearcher</groupId>
 	<artifactId>SubTitleSearcher</artifactId>
-	<version>2.0.3.0</version>
+	<version>2.0.4.0</version>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<copyright>Copyright 2019 pslib.com</copyright>

src/main/java/zimu/AppConfig.java

@@ -7,7 +7,7 @@ public class AppConfig {
 	public static String appName = "SubTitleSearcher";
 	public static String appTitle = "字幕下载";
 	//public static String appTitle = "SubTitleSearcher";
-	public static String appVer = "2.0.3";
+	public static String appVer = "2.0.4";
 
 	public static String appPath;
 	public static boolean isExe;

src/main/java/zimu/common/sites/ZIMuKuCommon.java

@@ -9,12 +9,14 @@
 
 import cn.hutool.core.codec.Base64;
 import cn.hutool.core.util.CharsetUtil;
+import cn.hutool.core.util.StrUtil;
 import cn.hutool.core.util.URLUtil;
 import cn.hutool.http.HttpResponse;
 import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.log.Log;
 import cn.hutool.log.LogFactory;
+import zimu.util.ExeJsUtil;
 import zimu.util.HtHttpUtil;
 import zimu.util.StringUtil;
 import zimu.util.regex.RegexUtil;
@@ -33,9 +35,9 @@ public class ZIMuKuCommon {
 
 	public static void main(String[] args) throws Exception {
 		//System.out.println(DownList("憨豆特工.mkv"));
-		System.out.println(DownList("downsizing.2017.720p.bluray.x264-geckos.mkv"));
-		
-		
+		//System.out.println(DownList("downsizing.2017.720p.bluray.x264-geckos.mkv"));
+		System.out.println(DownList("From.Beijing.with.Love.1994.720p.BluRay.x264-WiKi.mkv"));
+		//System.out.println(getPageList("From.Beijing.with.Love"));
 
 		//System.out.println(downContent("/detail/101779.html"));;
 		//detail/100250.html
@@ -67,13 +69,13 @@ public static JSONArray DownList(String fileName) throws Exception {
 	 * @return
 	 */
 	public static JSONObject downContent(String url) {
-		String result = HtHttpUtil.http.get(baseUrl+url);
+		String result = httpGet(baseUrl+url);
 		String downUrl = RegexUtil.getMatchStr(result, 
 				"<a\\s+id=\"down1\"\\s+href=\"(/dld/[\\w]+\\.html)\""
 				, Pattern.DOTALL);
 		if(downUrl == null)return null;
 		downUrl = baseUrl + downUrl;
-		result = HtHttpUtil.http.get(downUrl);
+		result = httpGet(downUrl);
 		if(result == null)return null;
 		//System.out.println(result);
 		JSONArray resList = RegexUtil.getMatchList(result, 
@@ -102,7 +104,7 @@ public static JSONObject downContent(String url) {
 	 * @return
 	 */
 	public static JSONArray getDetailList(String url) {
-		String result = HtHttpUtil.http.get(baseUrl+url);
+		String result = httpGet(baseUrl+url);
 		//System.out.println(result);
 		Document doc = Jsoup.parse(result);
 		Elements matchList = doc.select("#subtb tbody tr");
@@ -189,7 +191,7 @@ public static JSONArray getFuzzyPageList(String title) {
 	 * @return
 	 */
 	public static JSONArray getPageList(String title) {
-		String result = HtHttpUtil.http.get(baseUrl+"/search?q="+URLUtil.encodeAll(title, CharsetUtil.CHARSET_UTF_8));
+		String result = httpGet(baseUrl+"/search?q="+URLUtil.encodeAll(title, CharsetUtil.CHARSET_UTF_8));
 		//System.out.println(result);
 		JSONArray resList = RegexUtil.getMatchList(result, "<p\\s+class=\"tt\\s+clearfix\"><a\\s+href=\"(/subs/[\\w]+\\.html)\"\\s+"
 				+ "target=\"_blank\"><b>(.*?)</b></a></p>", Pattern.DOTALL);
@@ -200,4 +202,26 @@ public static JSONArray getPageList(String title) {
 
 		return resList;
 	}
+	
+	public static String httpGet(String url) {
+		String result = HtHttpUtil.http.get(url);
+		if(result!=null && StrUtil.count(result, "url")>10 && result.contains("<script")) {
+			String jsStr = RegexUtil.getMatchStr(result, "<script[^>]*>(.*?)</script>");
+			jsStr = jsStr.replaceAll("window.location[\\s]*=[\\s]*url", "");
+			jsStr = jsStr.replaceAll("location[\\s]*=[\\s]*url", "");
+			if(jsStr==null) {
+				jsStr = "";
+			}
+			String jsVal = null;
+			try {
+				jsVal = ExeJsUtil.getJsVal("function getUrl(){"+jsStr+";return url;} getUrl()");
+			}catch(Exception e) {
+				logger.error(e);
+			}
+			if(jsVal!=null&&jsVal.length()>0) {
+				return httpGet(jsVal.contains("://") ? jsVal : baseUrl+jsVal);
+			}
+		}
+		return result;
+	}
 }

src/main/java/zimu/tests/Test2.java

@@ -0,0 +1,24 @@
+package zimu.tests;
+
+import javax.script.Compilable;
+import javax.script.CompiledScript;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+
+public class Test2 {
+
+	public static void main(String[] args) {
+		ScriptEngineManager manager = new ScriptEngineManager();
+		ScriptEngine engine = manager.getEngineByName("JavaScript");
+		try {
+			String script = "function getUrl(){var url='adsf';return url;} getUrl()";
+			Compilable compilable = (Compilable) engine;
+			CompiledScript JSFunction = compilable.compile(script);
+			Object result = JSFunction.eval();
+			System.out.println(result);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+}

src/main/java/zimu/util/ExeJsUtil.java

@@ -0,0 +1,28 @@
+package zimu.util;
+
+import javax.script.Compilable;
+import javax.script.CompiledScript;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+
+public class ExeJsUtil {
+	
+	/**
+	 * 执行js并返回结果
+	 * @param jsStr
+	 * @return
+	 */
+	public static String getJsVal(String jsStr) {
+		ScriptEngineManager manager = new ScriptEngineManager();
+		ScriptEngine engine = manager.getEngineByName("JavaScript");
+		try {
+			Compilable compilable = (Compilable) engine;
+			CompiledScript JSFunction = compilable.compile(jsStr);
+			Object result = JSFunction.eval();
+			return result != null ? result.toString() : null;
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		return null;
+	}
+}