Merge pull request #326 from maarten-boot/master

sync with 1.20230720.2

Author: maarten-boot

README.md

@@ -114,7 +114,8 @@ Raise an issue https://github.com/DannyCork/python-whois/issues/new
  * suppress messages to stderr if not verbose=True
 
 2023-07-20: maarten_boot
-* sync with https://github.com/mboot-github/WhoisDomain (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout)
+* sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.1; (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout)
+* sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.2; add t_test hint support; fix some server hints
 
 ## Support
  * Python 3.x is supported for x >= 6

whois/_0_init_tld.py

@@ -128,10 +128,10 @@ def buildRegCollection(zz: Dict[str, Any]) -> Dict[str, Any]:
             if key is None:
                 continue
 
-            if key.startswith("_"):
+            if key.startswith("_"):  # skip meta keys, they are not regexes
                 continue
 
-            if key in ["extend"]:
+            if key in ["extend"]:  # this actually should have been a meta key: "_extend"
                 continue
 
             if key not in regCollection:

whois/__init__.py

@@ -77,6 +77,7 @@
     "QuotaStrings",
     "QuotaStringsAdd",
     "cleanupWhoisResponse",
+    "getTestHint",
 ]
 
 WHOISDOMAIN: str = ""
@@ -178,6 +179,7 @@ def _doServerHintsForThisTld(
     server: Optional[str],
     verbose: bool = False,
 ) -> Optional[str]:
+    # note _server hints currently are not passes down when using "extend", that may have been my error during the initial implementation
     # allow server hints using "_server" from the tld_regexpr.py file
     thisTldServer = thisTld.get("_server")
     if server is None and thisTldServer:
@@ -495,3 +497,14 @@ def query(
 
 def getVersion() -> str:
     return VERSION
+
+
+def getTestHint(tld: str) -> Optional[str]:
+    if tld not in ZZ:
+        return None
+
+    k: str = "_test"
+    if k not in ZZ[tld]:
+        return None
+
+    return str(ZZ[tld][k])

whois/main.py

@@ -28,6 +28,7 @@
 
 Failures: Dict[str, Any] = {}
 IgnoreReturncode: bool = False
+TestAllTld: bool = False
 
 
 class ResponseCleaner:
@@ -209,6 +210,7 @@ def testItem(
     global PrintGetRawWhoisResult
     global SIMPLISTIC
     global WithRedacted
+    global TestAllTld
 
     timeout = 30  # seconds
 
@@ -350,13 +352,26 @@ def makeMetaAllCurrentTld(
     allHaving: Optional[str] = None,
     allRegex: Optional[str] = None,
 ) -> List[str]:
+    global TestAllTld
+
+    def appendHint(
+        allRegex: Optional[str],
+        tld: str,
+    ) -> None:
+        if TestAllTld is True:
+            hint = whois.getTestHint(tld)
+            hint = hint if hint else f"meta.{tld}"
+            rr.append(f"{hint}")
+        else:
+            rr.append(f"meta.{tld}")
+
     rr: List[str] = []
     for tld in getAllCurrentTld():
         if allRegex:
             if re.search(allRegex, tld):
-                rr.append(f"meta.{tld}")
+                appendHint(allRegex, tld)
         else:
-            rr.append(f"meta.{tld}")
+            appendHint(allRegex, tld)
 
     return rr
 
@@ -478,6 +493,7 @@ def main() -> None:
     global Ruleset
     global SIMPLISTIC
     global WithRedacted
+    global TestAllTld
 
     name: str = os.path.basename(sys.argv[0])
     if name == "test2.py":
@@ -511,7 +527,7 @@ def main() -> None:
         usage()
         sys.exit(2)
 
-    testAllTld: bool = False
+    # TestAllTld: bool = False
 
     allHaving: Optional[str] = None  # from all supported tld only process the ones having this :: TODO ::
     allRegex: Optional[str] = None  # from all supported tld process only the ones matching this regex
@@ -545,14 +561,14 @@ def main() -> None:
             WithRedacted = True
 
         if opt in ("-a", "--all"):
-            testAllTld = True
+            TestAllTld = True
 
         if opt in ("-H", "--having"):
-            testAllTld = True
+            TestAllTld = True
             allHaving = str(arg)
 
         if opt in ("-r", "--reg"):
-            testAllTld = True
+            TestAllTld = True
             allRegex = str(arg)
 
         if opt in ("-v", "--verbose"):
@@ -595,7 +611,7 @@ def main() -> None:
 
             if filename not in files:
                 files.append(filename)
-                testAllTld = False
+                TestAllTld = False
 
         if opt in ("-d", "--domain"):
             domain = arg
@@ -610,7 +626,7 @@ def main() -> None:
             ShowRuleset(domain)
         sys.exit(0)
 
-    if testAllTld:
+    if TestAllTld:
         allMetaTld = makeMetaAllCurrentTld(allHaving, allRegex)
         testDomains(allMetaTld)
         showFailures()

whois/tld_regexpr.py

@@ -298,22 +298,21 @@
     "creation_date": r"Created on\.+:\s?(.+).",
     "expiration_date": r"Expires on\.+:\s?(.+).",  # note the trailing . on both dates fields
     "updated_date": "",
-    # "name_servers": r"\*\* Domain Servers:\n(?:(\S+)\n)(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)\n?",
     "name_servers": r"\*\* Domain Servers:\n(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?",  # allow for ip addresses after the name server
     "status": None,
     "_server": "whois.trabis.gov.tr",
-    # "_test": "googl.com.tr"
+    "_test": "google.com.tr",
 }
 
 ZZ["gov.tr"] = {
     "extend": "com.tr",
-    # "name_servers": r"\*\* Domain Servers:\n(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?", # this has ip addresses after the nameserver
-    # "_test": "www.turkiye.gov.tr",
+    "_server": "whois.trabis.gov.tr",
+    "_test": "www.turkiye.gov.tr",
 }
 
-ZZ["edu.tr"] = {"extend": "com.tr"}
-ZZ["org.tr"] = {"extend": "com.tr"}
-ZZ["net.tr"] = {"extend": "com.tr"}
+ZZ["edu.tr"] = {"extend": "com.tr", "_server": "whois.trabis.gov.tr", "_test": "anadolu.edu.tr"}
+ZZ["org.tr"] = {"extend": "com.tr", "_server": "whois.trabis.gov.tr", "_test": "dergipark.org.tr"}
+ZZ["net.tr"] = {"extend": "com.tr", "_server": "whois.trabis.gov.tr", "_test": "trt.net.tr"}
 
 
 ZZ["co.il"] = {

whois/version.py

@@ -1 +1 @@
-VERSION = "1.20230718.3"
+VERSION = "1.20230720.2"