Merge pull request #241 from maarten-boot/development

add several tld,s; add testprogram maketestdatAll for fast investigation of new tld's

Author: MooCow

.gitignore

@@ -70,4 +70,5 @@ reformat-code.sh
 t1.py
 typescript
 test.out
+diff.out
 tmp/

DONE

@@ -0,0 +1,42 @@
+DONE
+2022-11-17:mboot;
+
+  - tld ac; add registrant_country
+
+  - email regex use \. in:  "emails": r"[\w\.-]+@[\w\.-]+\.[\w]{2,4}"
+    this now results in valid data on google.com and meta.com and all derived from .com tld
+
+  - add comment in tld_resgepr.py that emails, status, name_serevers are multi items (lists)
+    and all the rest are single results.
+
+  - add comments to beginning of tld_regexpr.py explaining that all matches are actually case insensitive (findall)
+    and that many whois responses have trailing whitespace and may end in \r\n
+    this helps with constructing regexes for future use
+
+  - add skipFromHere in _2_parse.py: lines starting with ^>>> signify the end of a normal whois response
+    after this line there is only human or legal information so we can simply skip that text
+    (a similar construct with ^--\s will be done later)
+    (( retracted we have domains where this is very early: switched off for now, make this configurable ))
+
+  - add commment that unfortunately we cannot currently use rtrim on input from whois response and may regexes expect
+    either \r or trailing whitespace, this can be done later and would make many regexes simpler in end detection
+
+  - simplefy com.tr regex from multiple . to \.+, and add note on the trailing dot behind the date
+    add meta.com.tr to regular testdata
+
+  - fix test2.py cleanpostabmble for e.g com.sg which has lines at the top with ------------
+  - add getops to makeTestDataAll.sh
+  - allow maketestDataAll.sh -v -f -d com.sg to show the reduced input.out and the produced output of test2.py
+  - add '[WHITESPACE AT END]' in input.out during input reduction test
+
+  - add tld do, com.do, mo, com.mo, cx, dz (never answeres), gd, mn, gay, tl, tt (responds with perl script)
+
+  - add -t <tld> and -d <dmian> to makeTastdataAll support a cusom domain and not meta, google default
+
+  - add tld st, so, nrw, lat, so , realestate, ph, com.ph, org.ph, net.ph, zm
+
+  -  fix tokyo, it needs a explicit server, add sy it has a whois server but i cannot get it to answer, set to privateReg
+  -  add net.tr and tr (privateReg), onl, blue, garden, promo, one,
+
+  -  add nic to default test group for makeTestdataAll.sh
+

bin/find_input_no_output.sh

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+get_status_output()
+{
+    ls ./tmp/ |
+    while read item
+    do
+        d="tmp/$item"
+        [ -s "$d/input" ] && {
+            [ ! -f "$d/output" ] && {
+                echo "# NO_OUTPUT for $item"
+            }
+        }
+    done
+}
+    
+main()
+{
+    get_status_output
+}
+
+main

makeTestdataAll.sh

@@ -2,12 +2,13 @@
 
 TMPDIR="./tmp" # the default work directory is a local tmp (exclude by .gitignore )
 
-FORCE=0 # force whois loogup by not using cached data
-VERBOSE=0 # along the way inform us on progress and results
+FORCE=0     # force whois loogup by not using cached data
+VERBOSE=0   # along the way inform us on progress and results
+ALL=0       # process all supported tld's
 
 prepTempDir()
 {
-    # make the work dir if it does nt exist
+    # make the work dir if it does not exist
     mkdir -p "$TMPDIR" || {
         echo "FATAL: cannot make test dir: $TMPDIR" >&2
         exit 101
@@ -27,23 +28,27 @@ testNameserverExistsInInputAndOutput()
     local dns="$d/__dns-ns"
 
     [ -s "$dns" ] || return # dont bother at all if we have no dns file
+    [ -s "$d/input" ] || return # dont bother if we have no input file
 
     rm -f "$d/error.ns"
 
     cat "$dns" |
     awk '{ print $NF }' |
     while read ns
     do
-        [ -s "$d/input" ] || return # dont bother if we have no input file
-
+        [ "$VERBOSE" = "1" ] && echo "test $ns in input"
         grep -q -i "$ns" "$d/input" && {
             # only test in the output if it is present in the input
+
+            [ "$VERBOSE" = "1" ] && echo "test $ns in output"
             grep -q -i "$ns" "$d/output" || {
                 echo "ERROR: output;  missing nameserver '$ns' for tld: $tld" |
                 tee -a "$d/error.ns"
+                return 1
             }
         }
     done
+    return 0
 }
 
 cleanupTldTestDirectory()
@@ -128,7 +133,6 @@ getDnsSoaRecordAndLeaveEvidenceTldDomain()
 makeDirectoryForTld()
 {
     local tld="$1"
-    local domain="$2"
     local d="$TMPDIR/$tld"
 
     mkdir -p "$d" || {
@@ -153,48 +157,63 @@ makeTestDataOriginalOneTldDomain()
 
     getDnsSoaRecordAndLeaveEvidenceTldDomain "$tld" "$domain" || return 1
 
-    # what domain did we test
-    touch "$d/__domain__$zz"
-
-    # store the nameservers from dns
-    host -t ns "$zz" > "$d/__dns-ns__$zz"
-
+    touch "$d/__domain__$zz"                # what domain did we test
+    host -t ns "$zz" > "$d/__dns-ns__$zz"   # store the nameservers from dns
     getTestDataInputForTldAndDomain "$tld" "$domain" || return 1
-
-    getTestDataOutputForTldAndDomain "$tld" "$domain"
+    getTestDataOutputForTldAndDomain "$tld" "$domain" || return 1
     return 0
 }
 
 domainsToTry()
 {
-    cat <<! |
+    local domain="$1"
+
+    [ "$domain" = "__DEFAULT__" ] && {
+        cat <<! |
+nic
 meta
 google
 !
-    awk  '
-    /^[ \t]*$/ { next }
-    /^[ \t]*;/ { next }
-    /^[ \t]*#/ { next }
-    { print $1 }
-    '
+        awk  '
+        /^[ \t]*$/ { next }
+        /^[ \t]*;/ { next }
+        /^[ \t]*#/ { next }
+        { print $1 }
+        '
+        return
+    }
+
+    echo "$domain"
 }
 
 makeTestDataTldFromDomains()
 {
     local tld="$1"
+    local domain="$2"
 
-    domainsToTry |
+    domainsToTry "$domain" |
     while read domain
     do
         [ "$VERBOSE" = "1" ] && echo "try: $domain.$tld"
 
-        makeTestDataOriginalOneTldDomain "$tld" "$domain"
-
-        [ -s "$TMPDIR/$tld/input" ] && {
-            [ "$VERBOSE" = "1" ] && ls -l "$TMPDIR/$tld/"
-            testNameserverExistsInInputAndOutput "$tld" && break
+        makeTestDataOriginalOneTldDomain "$tld" "$domain" && {
+            [ -s "$TMPDIR/$tld/input" ] && {
+                [ "$VERBOSE" = "1" ] && {
+                    ls -l "$TMPDIR/$tld/"
+                }
+            }
+            break
         }
     done
+
+    [ "$VERBOSE" = "1" ] && {
+        # show the rediced input and the output
+        cat "$TMPDIR/$tld/input.out" "$TMPDIR/$tld/output"
+    }
+
+    testNameserverExistsInInputAndOutput "$tld" && {
+        return
+    }
 }
 
 makeRulesFromTldIfExist()
@@ -209,12 +228,13 @@ makeRulesFromTldIfExist()
 makeTestDataOriginalOneTld()
 {
     local tld="$1"
+    local domain="$2"
 
     [ "$VERBOSE" = "1" ] && echo "try: $tld"
 
-    makeDirectoryForTld "$tld" "$domain" || exit 101
+    makeDirectoryForTld "$tld" || exit 101
     makeRulesFromTldIfExist "$tld"
-    makeTestDataTldFromDomains "$tld"
+    makeTestDataTldFromDomains "$tld" "$domain"
 }
 
 makeTestDataOriginalAllTldSupported()
@@ -227,19 +247,62 @@ makeTestDataOriginalAllTldSupported()
     done
 }
 
+usage()
+{
+    cat <<!
+$0 usage:
+-h  show the help text
+-v  switch on verbose (will show progress)
+-f  switch on force (will re analyze all)
+-t  <domain> specify a tld to analize
+-a  analyze all tld currently supported
+!
+    exit 0;
+}
+
 main()
 {
+    [ $# -eq 0 ] && usage
+
+    VERBOSE=0
+    FORCE=0
+    ALL=0
+
+    local domain="__DEFAULT__"
+
+    while getopts "havft:d:" arg;
+    do
+        case $arg in
+
+        v) VERBOSE=1
+            ;;
+
+        f) FORCE=1
+            ;;
+
+        a) ALL=1
+            ;;
+
+        t) tld="${OPTARG}"
+            ;;
+
+        d) domain="${OPTARG}"
+            # instead of the default meta and google use this domain and combine it with the tld for processing
+            ;;
+
+        h | *) usage
+            ;;
+        esac
+    done
+
     prepTempDir
 
-    [ "$#" = "0" ] && {
+    [ "$ALL" = "1" ] && {
         makeTestDataOriginalAllTldSupported
         return
     }
 
-    for tld in $*
-    do
-        makeTestDataOriginalOneTld "$tld"
-    done
+    makeTestDataOriginalOneTld "$tld" "$domain"
 }
 
 main $* 2>&1 |

test.sh

@@ -22,7 +22,8 @@ testOneDomain()
     echo "testing: $domain"
     ./test2.py -d "$domain" >"$TestDataDir/$domain/test.out"
 
-    diff "$TestDataDir/$domain/output" "$TestDataDir/$domain/test.out" | tee "$TestDataDir/$domain/out"
+    diff "$TestDataDir/$domain/output" "$TestDataDir/$domain/test.out" | 
+    tee "$TestDataDir/$domain/diff.out"
 }
 
 main()

test2.py

@@ -120,7 +120,7 @@ def cleanupWhoisResponse(
                     percentSeen = True
 
             if postambleSeen is False:
-                if line.startswith("--") or line.startswith(">>> ") or line.startswith("Copyright notice"):
+                if line.startswith("-- ") or line.startswith(">>> ") or line.startswith("Copyright notice"):
                     postambleSeen = True
 
             if postambleSeen is True:
@@ -155,9 +155,10 @@ def printMe(self):
         if len(self.rDict[k]):
             n = 0
             for lines in self.rDict[k]:
-                tab = " [TAB] " if "\t" in lines else "-------"  # tabs are present in this section
-                cr = " [CR] " if "\r" in lines else "------"  # \r is present in this section
-                print(f"# ------------- {k} Section: {n} {cr}{tab}---------")
+                ws = " [WHITESPACE AT END] " if re.search(r"[ \t]+\r?\n", lines) else ""
+                tab = " [TAB] " if "\t" in lines else ""  # tabs are present in this section
+                cr = " [CR] " if "\r" in lines else ""  # \r is present in this section
+                print(f"# --- {k} Section: {n} {cr}{tab}{ws}")
                 n += 1
                 print(lines)
 

testdata/example.com/output

@@ -12,3 +12,4 @@ statuses           list              ['']
 dnssec             bool              False
 name_servers       list              []
 registrant         str               ''
+emails             list              ['']

testdata/example.net/output

@@ -12,3 +12,4 @@ statuses           list              ['']
 dnssec             bool              False
 name_servers       list              []
 registrant         str               ''
+emails             list              ['']

testdata/example.org/input

@@ -58,8 +58,8 @@ Name Server: a.iana-servers.net
 Name Server: b.iana-servers.net
 DNSSEC: signedDelegation
 URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf/
->>> Last update of WHOIS database: 2022-11-07T21:05:05Z <<<
+>>> Last update of WHOIS database: 2022-11-17T10:47:37Z <<<
 
 For more information on Whois status codes, please visit https://icann.org/epp
 
-Terms of Use: Access to Public Interest Registry WHOIS information is provided to assist persons in determining the contents of a domain name registration record in the Public Interest Registry registry database. The data in this record is provided by Public Interest Registry for informational purposes only, and Public Interest Registry does not guarantee its accuracy. This service is intended only for query-based access. You agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data to (a) allow, enable, or otherwise support the transmission by e-mail, telephone, or facsimile of mass unsolicited, commercial advertising or solicitations to entities other than the data recipient's own existing customers; or (b) enable high volume, automated, electronic processes that send queries or data to the systems of Registry Operator, a Registrar, or Donuts except as reasonably necessary to register domain names or modify existing registrations. All rights reserved. Public Interest Registry reserves the right to modify these terms at any time. By submitting this query, you agree to abide by this policy.  The Registrar of Record identified in this output may have an RDDS service that can be queried for additional information on how to contact the Registrant, Admin, or Tech contact of the queried domain name.
+Terms of Use: Access to Public Interest Registry WHOIS information is provided to assist persons in determining the contents of a domain name registration record in the Public Interest Registry registry database. The data in this record is provided by Public Interest Registry for informational purposes only, and Public Interest Registry does not guarantee its accuracy. This service is intended only for query-based access. You agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data to (a) allow, enable, or otherwise support the transmission by e-mail, telephone, or facsimile of mass unsolicited, commercial advertising or solicitations to entities other than the data recipient's own existing customers; or (b) enable high volume, automated, electronic processes that send queries or data to the systems of Registry Operator, a Registrar, or Identity Digital except as reasonably necessary to register domain names or modify existing registrations. All rights reserved. Public Interest Registry reserves the right to modify these terms at any time. By submitting this query, you agree to abide by this policy.  The Registrar of Record identified in this output may have an RDDS service that can be queried for additional information on how to contact the Registrant, Admin, or Tech contact of the queried domain name.

testdata/example.org/output

@@ -12,3 +12,4 @@ statuses           list              ['serverDeleteProhibited https://icann.org/
 dnssec             bool              True
 name_servers       list              ['a.iana-servers.net', 'b.iana-servers.net']
 registrant         str               'ICANN'
+emails             list              ['']