Merge pull request #238 from maarten-boot/development

Refactor and add getting the raw whois data also for unsupported tld's

Author: MooCow

.gitignore

@@ -70,3 +70,4 @@ reformat-code.sh
 t1.py
 typescript
 test.out
+tmp/

README.md

@@ -69,6 +69,10 @@ Raise an issue https://github.com/DannyCork/python-whois/issues/new
  * add testing against static known data in dir: ./testdata/<domain>/output
  * test.sh will test all domains in testdata without actually calling whois, the input data is instead read from testdata/<domain>/input
 
+2022-11-11: maarten_boot
+ * add support for returning the raw data from the whois command: flag include_raw_whois_text
+ * add support for handling unsupported domains via whois raw text only: flag return_raw_text_for_unsupported_tld
+
 ## Support
  * Python 3.x is supported.
  * Python 2.x IS NOT supported.

makeTestdataAll.sh

@@ -0,0 +1,104 @@
+#! /bin/bash
+
+TMPDIR="./tmp"
+FORCE=0
+VERBOSE=1
+
+prepTempDir()
+{
+    mkdir -p "$TMPDIR" || {
+        echo "FATAL: cannot make test dir: $TMPDIR" >&2
+        exit 101
+    }
+}
+
+getAllTldSupported()
+{
+    ./test2.py -S
+}
+
+makeTestDataOriginalOne()
+{
+    local tld="$1"
+    local domain="$2"
+
+    local d="$TMPDIR/$tld"
+    mkdir -p "$d" || {
+        echo "FATAL: cannot make tld directory: '$d'" >&2
+        exit 101
+    }
+
+    local zz="$domain.$tld"
+
+    host -t soa "$zz" |
+    tee "$d/__dns-soa" |
+    grep " has SOA record " || {
+        # no soa record so that domain does not exist, cleanup the test dir
+        rm -f "$d/input" "$d/output" "$d/__domain__$zz" "$d/__dns-soa" "$d/dns-ns"
+        echo "INFO: no domain for $zz" >&2
+        return 1
+    }
+
+    host -t ns "$zz" > "$d/__dns-ns"
+
+    # what domain did we test
+    touch "$d/__domain__$zz"
+
+    # force english, force no cache
+    [ ! -s "$d/input" -o "$FORCE" = "1" ] && {
+        LANG=EN whois --force-lookup "meta.$tld" >"$d/input" || {
+            # whois has a problem
+            rm -f "$d/input" "$d/output" "$d/__domain__$zz" "$d/__dns-soa" "$d/dns-ns"
+            return 1
+        }
+    }
+
+    [ ! -s "$d/output" -o "$FORCE" = "1" ] && {
+        ./test2.py -d "meta.$tld" >"$d/output"
+    }
+
+    return 0
+}
+
+domainsToTry()
+{
+    cat <<! |
+meta
+google
+!
+    awk  '
+    /^[ \t]*$/ { next }
+    /^[ \t]*;/ { next }
+    /^[ \t]*#/ { next }
+    { print $1 }
+    '
+}
+
+makeTestDataOriginalAll()
+{
+    getAllTldSupported |
+    while read tld
+    do
+        echo "try: $tld"
+
+        domainsToTry |
+        while read domain
+        do
+            makeTestDataOriginalOne "$tld" "$domain"
+            [ -f "$TMPDIR/$tld/input" ] && {
+                [ "$VERBOSE" ] && {
+                    ls -l "$TMPDIR/$tld/"
+                }
+                break
+            }
+        done
+    done
+}
+
+main()
+{
+    prepTempDir
+    makeTestDataOriginalAll
+}
+
+main

test.sh

@@ -1,9 +1,12 @@
 #! /usr/bin/bash
 
 # signal whois module that we are testing, this reads data from testdata/<domain>/in
-export TEST_WHOIS_PYTHON="1"
-
-TestDataDir="./testdata"
+prepPath()
+{
+    local xpath="$1"
+    TestDataDir=$( realpath "$xpath" )
+    export TEST_WHOIS_PYTHON="$TestDataDir"
+}
 
 get_testdomains()
 {
@@ -24,11 +27,16 @@ testOneDomain()
 
 main()
 {
+    prepPath "testdata" # set a default
+    [ -d "$1" ] && { # if a argument and is a dir use that for testing
+        prepPath "$1"
+    }
+
     get_testdomains |
     while read line
     do
         testOneDomain $(basename $line)
     done
 }
 
-main
+main $*

test2.py

@@ -226,8 +226,9 @@ def main(argv):
     try:
         opts, args = getopt.getopt(
             argv,
-            "pvIhaf:d:D:r:H:",
+            "SpvIhaf:d:D:r:H:",
             [
+                "SupportedTld",
                 "print",
                 "verbose",
                 "IgnoreReturncode",
@@ -259,9 +260,14 @@ def main(argv):
     fileData = {}
 
     for opt, arg in opts:
+        if opt in ("-S", "SupportedTld"):
+            for tld in sorted(whois.validTlds()):
+                print(tld)
+            sys.exit(0)
+
         if opt == "-h":
             usage()
-            sys.exit()
+            sys.exit(0)
 
         if opt in ("-a", "--all"):
             testAllTld = True

whois/_1_query.py

@@ -120,11 +120,12 @@ def testWhoisPythonFromStaticTestData(
     verbose: bool = False,
 ) -> str:
     domain = ".".join(dl)
-
-    pathToTestFile = f"./testdata/{domain}/input"
+    testDir = os.getenv("TEST_WHOIS_PYTHON")
+    pathToTestFile = f"{testDir}/{domain}/input"
     if os.path.exists(pathToTestFile):
-        with open(pathToTestFile, mode="r") as f:
-            return f.read()
+        with open(pathToTestFile, mode="rb") as f:  # switch to binary mode as that is what Popen uses
+            # make sure the data is treated exactly the same as the output of Popen
+            return f.read().decode(errors="ignore")
 
     raise WhoisCommandFailed("")
 
@@ -135,7 +136,7 @@ def _do_whois_query(
     server: Optional[str] = None,
     verbose: bool = False,
 ) -> str:
-    # TODO: if getenv[TEST_WHOIS_PYTON] fake whois by reading static data from a file
+    # if getenv[TEST_WHOIS_PYTON] fake whois by reading static data from a file
     # this wasy we can actually implemnt a test run with known data in and expected data out
     if os.getenv("TEST_WHOIS_PYTHON"):
         return testWhoisPythonFromStaticTestData(dl, ignore_returncode, server, verbose)
@@ -150,7 +151,6 @@ def _do_whois_query(
         env={"LANG": "en"} if dl[-1] in ".jp" else None,
     )
 
-    # print(p.stdout.read()+' '+p.stderr.read())
     r = p.communicate()[0].decode(errors="ignore")
     if ignore_returncode is False and p.returncode not in [0, 1]:
         raise WhoisCommandFailed(r)

whois/_3_adjust.py

@@ -36,10 +36,16 @@ def __init__(
         whois_str: Optional[str] = None,
         verbose: bool = False,
         include_raw_whois_text: bool = False,
+        return_raw_text_for_unsupported_tld: bool = False,
     ):
         if include_raw_whois_text and whois_str is not None:
             self.text = whois_str
 
+        if return_raw_text_for_unsupported_tld is True:
+            self.tld = data["tld"]
+            self.name = data["domain_name"][0].strip().lower()
+            return
+
         self.name = data["domain_name"][0].strip().lower()
         self.tld = data["tld"]