Merge pull request #232 from maarten-boot/temp2

move sections in do_parse to functions; prepare for handling old style IANA domain

Author: MooCow

test2.py

@@ -4,15 +4,21 @@
 import re
 import getopt
 import sys
+import json
 
 Verbose = False
 Failures = {}
 IgnoreReturncode = False
 
 
 def prepItem(d):
-    print("-" * 80)
-    print(d)
+    print("")
+    print(f"test domain: <<<<<<<<<< {d} >>>>>>>>>>>>>>>>>>>>")
+
+
+def xType(x):
+    s = f"{type(x)}"
+    return s.split("'")[1]
 
 
 def testItem(d):
@@ -27,9 +33,21 @@ def testItem(d):
         print("None")
         return
 
+    # the 3 date time items can be None if not present or a datetime string
+    # dnssec is a bool
+    # some strings are return as '' when empty (status)
+    # statuses can be a array of one empty string if no data
+
+    # not all values are always present it mainly depends on whet we see in the output of whois
+    # if we return not None: the elements that ars always there ars domain_name , tld, dnssec
+
     wd = w.__dict__
     for k, v in wd.items():
-        print('%20s\t"%s"' % (k, v))
+        ss = "%-18s %-17s "
+        if isinstance(v, str):
+            print((ss + "'%s'") % (k, xType(v), v))
+        else:
+            print((ss + "%s") % (k, xType(v), v))
 
 
 def errorItem(d, e, what="Generic"):
@@ -74,8 +92,8 @@ def testDomains(aList):
             errorItem(d, e, what="WhoisQuotaExceeded")
         except whois.WhoisPrivateRegistry as e:
             errorItem(d, e, what="WhoisPrivateRegistry")
-        except Exception as e:
-            errorItem(d, e, what="Generic")
+        # except Exception as e:
+        #    errorItem(d, e, what="Generic")
 
 
 def getTestFileOne(fPath, fileData):

whois/_2_parse.py

@@ -95,6 +95,90 @@ def cleanupWhoisResponse(
     return response
 
 
+def handleShortResponse(
+    tld: str,
+    dl: List,
+    whois_str: str,
+    verbose: bool = False,
+): # returns None or raises one of (WhoisQuotaExceeded, FailedParsingWhoisOutput)
+    if verbose:
+        d = ".".join(dl)
+        print(f"line count < 5:: {tld} {d} {whois_str}", file=sys.stderr)
+
+    s = whois_str.strip().lower()
+
+    # NOTE: from here s is lowercase only
+    # ---------------------------------
+    noneStrings = [
+        "not found",
+        "no entries found",
+        "status: free",
+        "no such domain",
+        "the queried object does not exist",
+        "domain you requested is not known",
+        "status: available",
+    ]
+
+    for i in noneStrings:
+        if i in s:
+            return None
+
+    # ---------------------------------
+    # is there any error string in the result
+    if s.count("error"):
+        return None
+
+    # ---------------------------------
+    quotaStrings = [
+        "limit exceeded",
+        "quota exceeded",
+        "try again later",
+        "please try again",
+        "exceeded the maximum allowable number",
+        "can temporarily not be answered",
+        "please try again.",
+        "queried interval is too short",
+    ]
+
+    for i in quotaStrings:
+        if i in s:
+            raise WhoisQuotaExceeded(whois_str)
+
+    # ---------------------------------
+    # ToDo:  Name or service not known
+
+    # ---------------------------------
+    raise FailedParsingWhoisOutput(whois_str)
+
+def doDnsSec(whois_str: str) -> bool:
+    whois_dnssec: Any = whois_str.split("DNSSEC:")
+    if len(whois_dnssec) >= 2:
+        whois_dnssec = whois_dnssec[1].split("\n")[0]
+        if whois_dnssec.strip() == "signedDelegation" or whois_dnssec.strip() == "yes":
+            return True
+    return False
+
+def doSourceIana(whois_str: str, verbose: bool = False) -> str:
+    # here we can handle the example.com and example.net permanent IANA domains
+
+    if verbose:
+        msg = f"i have seen source: IANA"
+        print(msg, file=sys.stderr)
+
+    whois_splitted = whois_str.split("source:       IANA")
+    if len(whois_splitted) == 2:
+        whois_str = whois_splitted[1] # often this is actually just whitespace
+    return whois_str
+
+def doIfServerNameLookForDomainName(whois_str: str, verbose: bool = False) -> str:
+    # not often available anymore
+    if re.findall(r"Server Name:\s?(.+)", whois_str, re.IGNORECASE):
+        if verbose:
+            msg = f"i have seen Server Name:, looking for Domain Name:"
+            print(msg, file=sys.stderr)
+        whois_str = whois_str[whois_str.find("Domain Name:") :]
+    return whois_str
+
 def do_parse(
     whois_str: str,
     tld: str,
@@ -111,84 +195,24 @@ def do_parse(
     )
 
     if whois_str.count("\n") < 5:
-        if verbose:
-            d = ".".join(dl)
-            print(f"line count < 5:: {tld} {d} {whois_str}", file=sys.stderr)
-
-        s = whois_str.strip().lower()
-
-        # NOTE: from here s is lowercase only
-        # ---------------------------------
-        noneStrings = [
-            "not found",
-            "no entries found",
-            "status: free",
-            "no such domain",
-            "the queried object does not exist",
-            "domain you requested is not known",
-            "status: available",
-        ]
-
-        for i in noneStrings:
-            if i in s:
-                return None
-
-        # ---------------------------------
-        # is there any error string in the result
-        if s.count("error"):
-            return None
+        return handleShortResponse(tld, dl, whois_str, verbose)
 
-        # ---------------------------------
-        quotaStrings = [
-            "limit exceeded",
-            "quota exceeded",
-            "try again later",
-            "please try again",
-            "exceeded the maximum allowable number",
-            "can temporarily not be answered",
-            "please try again.",
-            "queried interval is too short",
-        ]
-
-        for i in quotaStrings:
-            if i in s:
-                raise WhoisQuotaExceeded(whois_str)
-
-        # ---------------------------------
-        # ToDo:  Name or service not known
-
-        # ---------------------------------
-        raise FailedParsingWhoisOutput(whois_str)
-
-    # check the status of DNSSEC
-    r["DNSSEC"] = False
-    whois_dnssec: Any = whois_str.split("DNSSEC:")
-    if len(whois_dnssec) >= 2:
-        whois_dnssec = whois_dnssec[1].split("\n")[0]
-        if whois_dnssec.strip() == "signedDelegation" or whois_dnssec.strip() == "yes":
-            r["DNSSEC"] = True
+    r["DNSSEC"] = doDnsSec(whois_str) # check the status of DNSSEC
 
-    # this is mostly not available in many tld's anymore, should be investigated
-    # split whois_str to remove first IANA part showing info for TLD only
-    whois_splitted = whois_str.split("source:       IANA")
-    if len(whois_splitted) == 2:
-        whois_str = whois_splitted[1]
+    if "source:       IANA" in whois_str: # prepare for handling historical IANA domains
+        whois_str = doSourceIana(whois_str, verbose)
 
-    # also not available for many modern tld's
-    sn = re.findall(r"Server Name:\s?(.+)", whois_str, re.IGNORECASE)
-    if sn:
-        whois_str = whois_str[whois_str.find("Domain Name:") :]
+    if "Server Name" in whois_str: # handle old type Server Name (not very common anymore)
+        whois_str = doIfServerNameLookForDomainName(whois_str, verbose)
 
-    # return TLD_RE["com"] as default if tld not exists in TLD_RE
-    for k, v in TLD_RE.get(tld, TLD_RE["com"]).items():
-        if k.startswith("_"):
-            # skip meta element like: _server or _privateRegistry
+    for k, v in TLD_RE.get(tld, TLD_RE["com"]).items(): # use TLD_RE["com"] as default if a regex is missing
+        if k.startswith("_"): # skip meta element like: _server or _privateRegistry
             continue
 
+        # Historical: here we use 'empty string' as default, not None
         if v is None:
             r[k] = [""]
         else:
             r[k] = v.findall(whois_str) or [""]
-            # print("DEBUG: Keyval = " + str(r[k]))
 
     return r

whois/__init__.py

@@ -48,7 +48,6 @@
 Map2Underscore = {
     ".ac.uk": "ac_uk",
     ".co.uk": "co_uk",
-
     ".co.il": "co_il",
     # uganda
     ".ca.ug": "ca_ug",

whois/tld_regexpr.py

@@ -115,7 +115,7 @@
 au = {
     "extend": "com",
     "registrar": r"Registrar Name:\s?(.+)",
-    "updated_date": r"Last Modified:([^\n]*)", # fix empty LastModified
+    "updated_date": r"Last Modified:([^\n]*)",  # fix empty LastModified
 }
 
 ax = {