add return_raw_text_for_unsupported_tld to return only the raw text for unsupported domains

Author: rl-devops

README.md

@@ -69,6 +69,10 @@ Raise an issue https://github.com/DannyCork/python-whois/issues/new
  * add testing against static known data in dir: ./testdata/<domain>/output
  * test.sh will test all domains in testdata without actually calling whois, the input data is instead read from testdata/<domain>/input
 
+2022-11-11: maarten_boot
+ * add support for returning the raw data from the whois command: flag include_raw_whois_text
+ * add support for handling unsupported domains via whois raw text only: flag return_raw_text_for_unsupported_tld
+
 ## Support
  * Python 3.x is supported.
  * Python 2.x IS NOT supported.

makeTestdataAll.sh

@@ -0,0 +1,104 @@
+#! /bin/bash
+
+TMPDIR="./tmp"
+FORCE=0
+VERBOSE=1
+
+prepTempDir()
+{
+    mkdir -p "$TMPDIR" || {
+        echo "FATAL: cannot make test dir: $TMPDIR" >&2
+        exit 101
+    }
+}
+
+getAllTldSupported()
+{
+    ./test2.py -S
+}
+
+makeTestDataOriginalOne()
+{
+    local tld="$1"
+    local domain="$2"
+
+    local d="$TMPDIR/$tld"
+    mkdir -p "$d" || {
+        echo "FATAL: cannot make tld directory: '$d'" >&2
+        exit 101
+    }
+
+    local zz="$domain.$tld"
+
+    host -t soa "$zz" |
+    tee "$d/__dns-soa" |
+    grep " has SOA record " || {
+        # no soa record so that domain does not exist, cleanup the test dir
+        rm -f "$d/input" "$d/output" "$d/__domain__$zz" "$d/__dns-soa" "$d/dns-ns"
+        echo "INFO: no domain for $zz" >&2
+        return 1
+    }
+
+    host -t ns "$zz" > "$d/__dns-ns"
+
+    # what domain did we test
+    touch "$d/__domain__$zz"
+
+    # force english, force no cache
+    [ ! -s "$d/input" -o "$FORCE" = "1" ] && {
+        LANG=EN whois --force-lookup "meta.$tld" >"$d/input" || {
+            # whois has a problem
+            rm -f "$d/input" "$d/output" "$d/__domain__$zz" "$d/__dns-soa" "$d/dns-ns"
+            return 1
+        }
+    }
+
+    [ ! -s "$d/output" -o "$FORCE" = "1" ] && {
+        ./test2.py -d "meta.$tld" >"$d/output"
+    }
+
+    return 0
+}
+
+domainsToTry()
+{
+    cat <<! |
+meta
+google
+!
+    awk  '
+    /^[ \t]*$/ { next }
+    /^[ \t]*;/ { next }
+    /^[ \t]*#/ { next }
+    { print $1 }
+    '
+}
+
+makeTestDataOriginalAll()
+{
+    getAllTldSupported |
+    while read tld
+    do
+        echo "try: $tld"
+
+        domainsToTry |
+        while read domain
+        do
+            makeTestDataOriginalOne "$tld" "$domain"
+            [ -f "$TMPDIR/$tld/input" ] && {
+                [ "$VERBOSE" ] && {
+                    ls -l "$TMPDIR/$tld/"
+                }
+                break
+            }
+        done
+    done
+}
+
+main()
+{
+    prepTempDir
+    makeTestDataOriginalAll
+}
+
+main

test.sh

@@ -1,8 +1,12 @@
 #! /usr/bin/bash
 
 # signal whois module that we are testing, this reads data from testdata/<domain>/in
-TestDataDir=$(realpath ./testdata)
-export TEST_WHOIS_PYTHON="$TestDataDir"
+prepPath()
+{
+    local xpath="$1"
+    TestDataDir=$( realpath "$xpath" )
+    export TEST_WHOIS_PYTHON="$TestDataDir"
+}
 
 get_testdomains()
 {
@@ -23,11 +27,16 @@ testOneDomain()
 
 main()
 {
+    prepPath "testdata" # set a default
+    [ -d "$1" ] && { # if a argument and is a dir use that for testing
+        prepPath "$1"
+    }
+
     get_testdomains |
     while read line
     do
         testOneDomain $(basename $line)
     done
 }
 
-main
+main $*

test2.py

@@ -260,7 +260,7 @@ def main(argv):
     fileData = {}
 
     for opt, arg in opts:
-        if opt in ( "-S", "SupportedTld"):
+        if opt in ("-S", "SupportedTld"):
             for tld in sorted(whois.validTlds()):
                 print(tld)
             sys.exit(0)

whois/_3_adjust.py

@@ -36,10 +36,16 @@ def __init__(
         whois_str: Optional[str] = None,
         verbose: bool = False,
         include_raw_whois_text: bool = False,
+        return_raw_text_for_unsupported_tld: bool = False,
     ):
         if include_raw_whois_text and whois_str is not None:
             self.text = whois_str
 
+        if return_raw_text_for_unsupported_tld is True:
+            self.tld = data["tld"]
+            self.name = data["domain_name"][0].strip().lower()
+            return
+
         self.name = data["domain_name"][0].strip().lower()
         self.tld = data["tld"]
 

whois/__init__.py

@@ -208,8 +208,10 @@ def fromDomainStringToTld(domain: str, internationalized: bool, verbose: bool =
     return tld, d
 
 
-def validateWeKnowTheToplevelDomain(tld):  # may raise UnknownTld
+def validateWeKnowTheToplevelDomain(tld, return_raw_text_for_unsupported_tld: bool = False):  # may raise UnknownTld
     if tld not in TLD_RE.keys():
+        if return_raw_text_for_unsupported_tld:
+            return None
         a = f"The TLD {tld} is currently not supported by this package."
         b = "Use validTlds() to see what toplevel domains are supported."
         msg = f"{a} {b}"
@@ -244,6 +246,45 @@ def doSlowdownHintForThisTld(tld: str, thisTld, slow_down: int, verbose: bool =
     return slow_down
 
 
+def doUnsupportedTldAnyway(
+    tld: str,
+    dl: Dict,
+    ignore_returncode: bool = False,
+    slow_down: int = 0,
+    server: Optional[str] = None,
+    verbose: bool = False,
+):
+    include_raw_whois_text = True
+
+    # we will not hunt for possible valid first level domains as we have no actual feedback
+
+    whois_str = do_query(
+        dl=dl,
+        slow_down=slow_down,
+        ignore_returncode=ignore_returncode,
+        server=server,
+        verbose=verbose,
+    )
+
+    # we will only return minimal data
+    data = {
+        "tld": tld,
+        "domain_name": "",
+    }
+    data["domain_name"] =  [".".join(dl)] # note the fields are default all array, except tld
+
+    if verbose:
+        print(data, file=sys.stderr)
+
+    return Domain(
+        data=data,
+        whois_str=whois_str,
+        verbose=verbose,
+        include_raw_whois_text=include_raw_whois_text,
+        return_raw_text_for_unsupported_tld=True,
+    )
+
+
 def query(
     domain: str,
     force: bool = False,
@@ -255,6 +296,7 @@ def query(
     with_cleanup_results=False,
     internationalized: bool = False,
     include_raw_whois_text: bool = False,
+    return_raw_text_for_unsupported_tld: bool = False,
 ) -> Optional[Domain]:
     """
     force=True          Don't use cache.
@@ -270,15 +312,28 @@ def query(
     verbose:            if true, print relevant information on steps taken to standard error
     include_raw_whois_text:
                         if reqested the full response is also returned.
+    return_raw_text_for_unsupported_tld:
+                        if the tld is unsupported, just try it anyway but return only the raw text.
     """
 
     assert isinstance(domain, str), Exception("`domain` - must be <str>")
+    return_raw_text_for_unsupported_tld = bool(return_raw_text_for_unsupported_tld)
 
     tld, dl = fromDomainStringToTld(domain, internationalized, verbose)
     if tld is None:
         return None
 
-    thisTld = validateWeKnowTheToplevelDomain(tld)  # may raise UnknownTld
+    thisTld = validateWeKnowTheToplevelDomain(tld, return_raw_text_for_unsupported_tld)  # may raise UnknownTld
+    if thisTld is None:
+        return doUnsupportedTldAnyway(
+            tld,
+            dl,
+            ignore_returncode=ignore_returncode,
+            slow_down=slow_down,
+            server=server,
+            verbose=verbose,
+        )
+
     verifyPrivateREgistry(thisTld)  # may raise WhoisPrivateRegistry
     server = doServerHintsForThisTld(tld, thisTld, server, verbose)