Merge pull request #325 from maarten-boot/master

sync with mboot-github/WhoisDomain

Author: maarten-boot

README.md

@@ -32,7 +32,7 @@ This package will not support querying ip CIDR ranges or AS information
 
 ## Docker
  * docker pull mbootgithub/whoisdomain:latest
- 
+
 ## Help Wanted
 Your contributions are welcome, look for the Help wanted tag https://github.com/DannyCork/python-whois/labels/help%20wanted
 
@@ -113,6 +113,9 @@ Raise an issue https://github.com/DannyCork/python-whois/issues/new
  * whois.NoneStringsAdd(str) to add additional string for NoSuchDomainExists detection (whois.query() retuning None). whois.NoneStrings() lsts the current configured strings
  * suppress messages to stderr if not verbose=True
 
+2023-07-20: maarten_boot
+* sync with https://github.com/mboot-github/WhoisDomain (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout)
+
 ## Support
  * Python 3.x is supported for x >= 6
  * Python 2.x IS NOT supported.

whois/_1_query.py

@@ -4,6 +4,7 @@
 import os
 import platform
 import json
+import shutil
 from .exceptions import WhoisCommandFailed, WhoisCommandTimeout
 
 from typing import Dict, List, Optional, Tuple
@@ -13,6 +14,13 @@
 CACHE: Dict[str, Tuple[int, str]] = {}
 CACHE_MAX_AGE = 60 * 60 * 48  # 48h
 
+IS_WINDOWS = platform.system() == "Windows"
+
+if not IS_WINDOWS and shutil.which("stdbuf"):
+    STDBUF_OFF_CMD = ["stdbuf", "-o0"]
+else:
+    STDBUF_OFF_CMD = []
+
 
 def _cache_load(cf: str) -> None:
     if not os.path.isfile(cf):
@@ -88,7 +96,7 @@ def _makeWhoisCommandToRun(
     else:
         whList = [wh]
 
-    if platform.system() == "Windows":
+    if IS_WINDOWS:
         if wh == "whois":  # only if the use did not specify what whois to use
             if os.path.exists("whois.exe"):
                 wh = r".\whois.exe"
@@ -122,6 +130,7 @@ def _do_whois_query(
     server: Optional[str] = None,
     verbose: bool = False,
     timeout: Optional[float] = None,
+    parse_partial_response: bool = False,
     wh: str = "whois",
     simplistic: bool = False,
 ) -> str:
@@ -141,7 +150,8 @@ def _do_whois_query(
 
     # LANG=en is added to make the ".jp" output consist across all environments
     p = subprocess.Popen(
-        cmd,
+        # STDBUF_OFF_CMD needed to not lose data on kill
+        STDBUF_OFF_CMD + cmd,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         env={"LANG": "en"} if dl[-1] in ".jp" else None,
@@ -152,8 +162,12 @@ def _do_whois_query(
     except subprocess.TimeoutExpired:
         # Kill the child process & flush any output buffers
         p.kill()
-        p.communicate()
-        raise WhoisCommandTimeout(f"timeout: query took more then {timeout} seconds")
+        r = p.communicate()[0].decode(errors="ignore")
+        # In most cases whois servers returns partial domain data really fast
+        # after that delay occurs (probably intentional) before returning contact data.
+        # Add this option to cover those cases
+        if not parse_partial_response or not r:
+            raise WhoisCommandTimeout(f"timeout: query took more then {timeout} seconds")
 
     if verbose:
         print(r, file=sys.stderr)
@@ -187,6 +201,7 @@ def do_query(
     server: Optional[str] = None,
     verbose: bool = False,
     timeout: Optional[float] = None,
+    parse_partial_response: bool = False,
     wh: str = "whois",
     simplistic: bool = False,
 ) -> str:
@@ -216,6 +231,7 @@ def do_query(
                 server=server,
                 verbose=verbose,
                 timeout=timeout,
+                parse_partial_response=parse_partial_response,
                 wh=wh,
                 simplistic=simplistic,
             ),

whois/_2_parse.py

@@ -27,18 +27,18 @@
     "the domain has not been registered",
     "no match found for",
     "no matching record",
+    "no match",
     "not found",
     "no data found",
     "no entries found",
-    "status: free",
+    # "status: free", # we should not interprete the result if there is a result
     "no such domain",
     "the queried object does not exist",
     "domain you requested is not known",
-    "status: available",
+    # "status: available", # we should not interprete the result if there is a result
     "no whois server is known for this kind of object",
     "nameserver not found",
     "malformed request",  # this means this domain is not in whois as it is on top of a registered domain
-    "no match",
     "registration of this domain is restricted",
     "restricted",
     "this domain is currently available",
@@ -288,7 +288,7 @@ def cleanupWhoisResponse(
 
 
 def NoneStrings() -> List[str]:
-    return sorted(NONESTRINGS)
+    return NONESTRINGS
 
 
 def NoneStringsAdd(aString: str) -> None:
@@ -297,7 +297,7 @@ def NoneStringsAdd(aString: str) -> None:
 
 
 def QuotaStrings() -> List[str]:
-    return sorted(QUOTASTRINGS)
+    return QUOTASTRINGS
 
 
 def QuotaStringsAdd(aString: str) -> None:

whois/__init__.py

@@ -258,6 +258,7 @@ def _doOneLookup(
     with_cleanup_results: bool = False,
     include_raw_whois_text: bool = False,
     timeout: Optional[float] = None,
+    parse_partial_response: bool = False,
     wh: str = "whois",
     simplistic: bool = False,
     withRedacted: bool = False,
@@ -274,6 +275,7 @@ def _doOneLookup(
             server=server,
             verbose=verbose,
             timeout=timeout,
+            parse_partial_response=parse_partial_response,
             wh=wh,
             simplistic=simplistic,
         )
@@ -344,6 +346,7 @@ def query(
     include_raw_whois_text: bool = False,
     return_raw_text_for_unsupported_tld: bool = False,
     timeout: Optional[float] = None,
+    parse_partial_response: bool = False,
     cmd: str = "whois",
     simplistic: bool = False,
     withRedacted: bool = False,
@@ -366,6 +369,8 @@ def query(
     return_raw_text_for_unsupported_tld:
                         if the tld is unsupported, just try it anyway but return only the raw text.
     timeout:            timeout in seconds for the whois command to return a result.
+    parse_partial_response:
+                        try to parse partial response when cmd timed out (stdbuf should be in PATH for best results)
     cmd:                explicitly specify the path to the whois you want to use.
     simplistic:         when simplistic is True we return None for most exceptions and dont pass info why we have no data.
     withRedacted:       show redacted output , default no redacted data is shown
@@ -464,6 +469,7 @@ def query(
             with_cleanup_results=with_cleanup_results,
             include_raw_whois_text=include_raw_whois_text,
             timeout=timeout,
+            parse_partial_response=parse_partial_response,
             wh=wh,
             simplistic=simplistic,
             withRedacted=withRedacted,

whois/main.py

@@ -14,9 +14,8 @@
     Dict,
 )
 
-import whois  # to be compatible with dannycork
-
 # import whoisdomain as whois  # to be compatible with dannycork
+import whois
 
 # if we are not running as test2.py run in a simplistic way
 SIMPLISTIC: bool = False

whois/tld_regexpr.py

@@ -24,7 +24,7 @@
     "extend": None,
     "domain_name": r"Domain Name\s*:\s*(.+)",
     "registrar": r"Registrar:\s?(.+)",
-    "registrant": r"Registrant\s*Organi(?:s|z)ation:\s?(.+)",
+    "registrant": r"Registrant\s*Organi(?:s|z)ation:([^\n]*)",  # this seems to match Registrant Street: if Registrant Organization: is empty
     "registrant_country": r"Registrant Country:\s?(.+)",
     "creation_date": r"Creation Date:[ \t]*([^\n]*)",
     "expiration_date": r"(?:Expiry|Expiration) Date:[ \t]*([^\n]*)",  # Expiration Date
@@ -287,6 +287,8 @@
     "extend": "au",
 }
 
+ZZ["tr"] = {"extend": "_privateReg"}  # whois.nic.tr is an alias for whois.trabis.gov.tr.
+
 ZZ["com.tr"] = {
     "extend": "com",
     "domain_name": r"\*\* Domain Name:\s?(.+)",
@@ -296,16 +298,24 @@
     "creation_date": r"Created on\.+:\s?(.+).",
     "expiration_date": r"Expires on\.+:\s?(.+).",  # note the trailing . on both dates fields
     "updated_date": "",
-    "name_servers": r"\*\* Domain Servers:\n(?:(\S+)\n)(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)\n?",
+    # "name_servers": r"\*\* Domain Servers:\n(?:(\S+)\n)(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)\n?",
+    "name_servers": r"\*\* Domain Servers:\n(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?",  # allow for ip addresses after the name server
     "status": None,
+    "_server": "whois.trabis.gov.tr",
+    # "_test": "googl.com.tr"
 }
 
-ZZ["edu.tr"] = {"extend": "com.tr"}
+ZZ["gov.tr"] = {
+    "extend": "com.tr",
+    # "name_servers": r"\*\* Domain Servers:\n(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?(?:(\S+).*\n)?", # this has ip addresses after the nameserver
+    # "_test": "www.turkiye.gov.tr",
+}
 
+ZZ["edu.tr"] = {"extend": "com.tr"}
 ZZ["org.tr"] = {"extend": "com.tr"}
-
 ZZ["net.tr"] = {"extend": "com.tr"}
 
+
 ZZ["co.il"] = {
     "extend": "com",
     "domain_name": r"domain:\s*(.+)",
@@ -664,10 +674,14 @@
 }
 
 ZZ["lv"] = {
-    "extend": "ru",
-    "creation_date": r"Registered:\s*(.+)\n",
+    "extend": "com",
+    "domain_name": r"domain:\s*(.+)",
+    "creation_date": r"Registered:\s*(.+)\n",  # actually there seem to be no dates
     "updated_date": r"Changed:\s*(.+)\n",
+    "expiration_date": r"paid-till:\s*(.+)",
+    "name_servers": r"nserver:\s*(.+)",
     "status": r"Status:\s?(.+)",
+    "_server": "whois.nic.lv",
 }
 
 ZZ["me"] = {
@@ -893,28 +907,25 @@
 
 ZZ["ru"] = {
     "extend": "com",
-    "domain_name": r"\ndomain:\s*(.+)",
-    "creation_date": r"\ncreated:\s*(.+)",
-    "expiration_date": r"\npaid-till:\s*(.+)",
-    "name_servers": r"\nnserver:\s*(.+)",
-    "status": r"\nstate:\s*(.+)",
+    "domain_name": r"domain:\s*(.+)",
+    "creation_date": r"created:\s*(.+)",
+    "expiration_date": r"paid-till:\s*(.+)",
+    "name_servers": r"nserver:\s*(.+)",
+    "status": r"state:\s*(.+)",
+    "_server": "whois.tcinet.ru",
 }
+ZZ["com.ru"] = {"extend": "ru", "_server": "whois.nic.ru"}  # test: mining.com.ru
+
+# Russian city sub-domains
+ZZ["msk.ru"] = {"extend": "com.ru"}  # test with: mining.msk.ru
+ZZ["spb.ru"] = {"extend": "com.ru"}  # test with iac.spb.ru
 
 # Rossíyskaya Federátsiya) is the Cyrillic country code top-level domain for the Russian Federation,
 # In the Domain Name System it has the ASCII DNS name xn--p1ai.
 
-ZZ["ru.rf"] = {
-    "extend": "ru",
-    "_server": "whois.tcinet.ru",
-}
-ZZ["рф"] = {
-    "extend": "ru",
-    "_server": "whois.tcinet.ru",
-}
-ZZ["xn--p1ai"] = {
-    "extend": "ru",
-    "_server": "whois.tcinet.ru",
-}
+ZZ["ru.rf"] = {"extend": "ru"}
+ZZ["рф"] = {"extend": "ru"}
+ZZ["xn--p1ai"] = {"extend": "ru"}
 
 ZZ["sa"] = {
     "extend": "com",
@@ -1161,23 +1172,10 @@
 ZZ["net.za"] = {"extend": "za", "_server": "net-whois.registry.net.za"}
 ZZ["co.za"] = {"extend": "za", "_server": "coza-whois.registry.net.za"}
 
-# Kenia
-# ke : http://www.kenic.or.ke/index.php/en/ke-domains/ke-domains
-ZZ["ke"] = {"extend": "com", "_server": "whois.kenic.or.ke"}
-ZZ["ac.ke"] = {"extend": "ke"}
-ZZ["co.ke"] = {"extend": "ke"}
-ZZ["go.ke"] = {"extend": "ke"}
-ZZ["info.ke"] = {"extend": "ke"}
-ZZ["me.ke"] = {"extend": "ke"}
-ZZ["mobi.ke"] = {"extend": "ke"}
-ZZ["ne.ke"] = {"extend": "ke"}
-ZZ["or.ke"] = {"extend": "ke"}
-ZZ["sc.ke"] = {"extend": "ke"}
 ZZ["gy"] = {"extend": "com"}
 
 # Multiple initialization
 ZZ["ca"] = {"extend": "bank"}
-
 # Rwanda: https://en.wikipedia.org/wiki/.rw
 ZZ["rw"] = {"extend": "com", "_server": "whois.ricta.org.rw"}
 ZZ[".co.rw"] = {"extend": "rw"}
@@ -1957,6 +1955,19 @@
 ZZ["com.py"] = {"extend": "_privateReg"}
 ZZ["sr"] = {"extend": "_privateReg"}
 
+# Kenia
+# ke : http://www.kenic.or.ke/index.php/en/ke-domains/ke-domains
+ZZ["ke"] = {"extend": "com", "_server": "whois.kenic.or.ke"}
+ZZ["ac.ke"] = {"extend": "ke"}
+ZZ["co.ke"] = {"extend": "ke"}
+ZZ["go.ke"] = {"extend": "ke"}
+ZZ["info.ke"] = {"extend": "ke"}
+ZZ["me.ke"] = {"extend": "ke"}
+ZZ["mobi.ke"] = {"extend": "ke"}
+ZZ["ne.ke"] = {"extend": "ke"}
+ZZ["or.ke"] = {"extend": "ke"}
+ZZ["sc.ke"] = {"extend": "ke"}
+
 # https://www.iana.org/domains/root/db/td.html
 # td = {"extend": "_privateReg"} # Chad (French: Tchad) made available for use in 1997.
 
@@ -2073,7 +2084,6 @@
 ZZ["net.ph"] = {"extend": "ph"}
 ZZ["zm"] = {"extend": "com"}
 ZZ["sy"] = {"extend": "_privateReg", "_server": "whois.tld.sy"}
-ZZ["tr"] = {"extend": "_privateReg"}
 ZZ["onl"] = {"extend": "com"}
 ZZ["blue"] = {"extend": "com"}
 ZZ["garden"] = {"extend": "com", "_server": "whois.nic.garden"}

whois/version.py

@@ -1 +1 @@
-VERSION = "1.20230525.9"
+VERSION = "1.20230718.3"