add new test domains for test.sh; add DONE reporting what was done in this update; fix email regex; add skip all text after >>>

Author: rl-devops

.gitignore

@@ -70,4 +70,5 @@ reformat-code.sh
 t1.py
 typescript
 test.out
+diff.out
 tmp/

DONE

@@ -0,0 +1,22 @@
+DONE
+2022-11-17:mboot;
+
+  - tld ac; add registrant_country
+
+  - email regex use \. in:  "emails": r"[\w\.-]+@[\w\.-]+\.[\w]{2,4}"
+    this now results in valid data on google.com and meta.com and all derived from .com tld
+
+  - add comment in tld_resgepr.py that emails, status, name_serevers are multi items (lists)
+    and all the rest are single results.
+
+  - add comments to beginning of tld_regexpr.py explaining that all matches are actually case insensitive (findall)
+    and that many whois responses have trailing whitespace and may end in \r\n
+    this helps with constructing regexes for future use
+
+  - add skipFromHere in _2_parse.py: lines starting with ^>>> signify the end of a normal whois response
+    after this line there is only human or legal information so we can simply skip that text
+    (a similar construct with ^--\s will be done later)
+
+  - add commment that unfortunately we cannot currently use rtrim on input from whois response and may regexes expect
+    either \r or trailing whitespace, this can be done later and would make many regexes simpler in end detection
+

bin/find_input_no_output.sh

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+get_status_output()
+{
+    ls ./tmp/ |
+    while read item
+    do
+        d="tmp/$item"
+        [ -s "$d/input" ] && {
+            [ ! -f "$d/output" ] && {
+                echo "# NO_OUTPUT for $item"
+            }
+        }
+    done
+}
+    
+main()
+{
+    get_status_output
+}
+
+main

test.sh

@@ -22,7 +22,8 @@ testOneDomain()
     echo "testing: $domain"
     ./test2.py -d "$domain" >"$TestDataDir/$domain/test.out"
 
-    diff "$TestDataDir/$domain/output" "$TestDataDir/$domain/test.out" | tee "$TestDataDir/$domain/out"
+    diff "$TestDataDir/$domain/output" "$TestDataDir/$domain/test.out" | 
+    tee "$TestDataDir/$domain/diff.out"
 }
 
 main()

testdata/example.com/output

@@ -12,3 +12,4 @@ statuses           list              ['']
 dnssec             bool              False
 name_servers       list              []
 registrant         str               ''
+emails             list              ['']

testdata/example.net/output

@@ -12,3 +12,4 @@ statuses           list              ['']
 dnssec             bool              False
 name_servers       list              []
 registrant         str               ''
+emails             list              ['']

testdata/example.org/input

@@ -58,8 +58,8 @@ Name Server: a.iana-servers.net
 Name Server: b.iana-servers.net
 DNSSEC: signedDelegation
 URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf/
->>> Last update of WHOIS database: 2022-11-07T21:05:05Z <<<
+>>> Last update of WHOIS database: 2022-11-17T10:47:37Z <<<
 
 For more information on Whois status codes, please visit https://icann.org/epp
 
-Terms of Use: Access to Public Interest Registry WHOIS information is provided to assist persons in determining the contents of a domain name registration record in the Public Interest Registry registry database. The data in this record is provided by Public Interest Registry for informational purposes only, and Public Interest Registry does not guarantee its accuracy. This service is intended only for query-based access. You agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data to (a) allow, enable, or otherwise support the transmission by e-mail, telephone, or facsimile of mass unsolicited, commercial advertising or solicitations to entities other than the data recipient's own existing customers; or (b) enable high volume, automated, electronic processes that send queries or data to the systems of Registry Operator, a Registrar, or Donuts except as reasonably necessary to register domain names or modify existing registrations. All rights reserved. Public Interest Registry reserves the right to modify these terms at any time. By submitting this query, you agree to abide by this policy.  The Registrar of Record identified in this output may have an RDDS service that can be queried for additional information on how to contact the Registrant, Admin, or Tech contact of the queried domain name.
+Terms of Use: Access to Public Interest Registry WHOIS information is provided to assist persons in determining the contents of a domain name registration record in the Public Interest Registry registry database. The data in this record is provided by Public Interest Registry for informational purposes only, and Public Interest Registry does not guarantee its accuracy. This service is intended only for query-based access. You agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data to (a) allow, enable, or otherwise support the transmission by e-mail, telephone, or facsimile of mass unsolicited, commercial advertising or solicitations to entities other than the data recipient's own existing customers; or (b) enable high volume, automated, electronic processes that send queries or data to the systems of Registry Operator, a Registrar, or Identity Digital except as reasonably necessary to register domain names or modify existing registrations. All rights reserved. Public Interest Registry reserves the right to modify these terms at any time. By submitting this query, you agree to abide by this policy.  The Registrar of Record identified in this output may have an RDDS service that can be queried for additional information on how to contact the Registrant, Admin, or Tech contact of the queried domain name.

testdata/example.org/output

@@ -12,3 +12,4 @@ statuses           list              ['serverDeleteProhibited https://icann.org/
 dnssec             bool              True
 name_servers       list              ['a.iana-servers.net', 'b.iana-servers.net']
 registrant         str               'ICANN'
+emails             list              ['']

testdata/google.com/input

@@ -0,0 +1,83 @@
+[Querying whois.verisign-grs.com]
+[Redirected to whois.markmonitor.com]
+[Querying whois.markmonitor.com]
+[whois.markmonitor.com]
+Domain Name: google.com
+Registry Domain ID: 2138514_DOMAIN_COM-VRSN
+Registrar WHOIS Server: whois.markmonitor.com
+Registrar URL: http://www.markmonitor.com
+Updated Date: 2019-09-09T15:39:04+0000
+Creation Date: 1997-09-15T07:00:00+0000
+Registrar Registration Expiration Date: 2028-09-13T07:00:00+0000
+Registrar: MarkMonitor, Inc.
+Registrar IANA ID: 292
+Registrar Abuse Contact Email: [email protected]
+Registrar Abuse Contact Phone: +1.2086851750
+Domain Status: clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)
+Domain Status: clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)
+Domain Status: clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)
+Domain Status: serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)
+Domain Status: serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)
+Domain Status: serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)
+Registrant Organization: Google LLC
+Registrant State/Province: CA
+Registrant Country: US
+Registrant Email: Select Request Email Form at https://domains.markmonitor.com/whois/google.com
+Admin Organization: Google LLC
+Admin State/Province: CA
+Admin Country: US
+Admin Email: Select Request Email Form at https://domains.markmonitor.com/whois/google.com
+Tech Organization: Google LLC
+Tech State/Province: CA
+Tech Country: US
+Tech Email: Select Request Email Form at https://domains.markmonitor.com/whois/google.com
+Name Server: ns2.google.com
+Name Server: ns3.google.com
+Name Server: ns1.google.com
+Name Server: ns4.google.com
+DNSSEC: unsigned
+URL of the ICANN WHOIS Data Problem Reporting System: http://wdprs.internic.net/
+>>> Last update of WHOIS database: 2022-11-17T10:39:21+0000 <<<
+
+For more information on WHOIS status codes, please visit:
+  https://www.icann.org/resources/pages/epp-status-codes
+
+If you wish to contact this domain’s Registrant, Administrative, or Technical
+contact, and such email address is not visible above, you may do so via our web
+form, pursuant to ICANN’s Temporary Specification. To verify that you are not a
+robot, please enter your email address to receive a link to a page that
+facilitates email communication with the relevant contact(s).
+
+Web-based WHOIS:
+  https://domains.markmonitor.com/whois
+
+If you have a legitimate interest in viewing the non-public WHOIS details, send
+your request and the reasons for your request to [email protected]
+and specify the domain name in the subject line. We will review that request and
+may ask for supporting documentation and explanation.
+
+The data in MarkMonitor’s WHOIS database is provided for information purposes,
+and to assist persons in obtaining information about or related to a domain
+name’s registration record. While MarkMonitor believes the data to be accurate,
+the data is provided "as is" with no guarantee or warranties regarding its
+accuracy.
+
+By submitting a WHOIS query, you agree that you will use this data only for
+lawful purposes and that, under no circumstances will you use this data to:
+  (1) allow, enable, or otherwise support the transmission by email, telephone,
+or facsimile of mass, unsolicited, commercial advertising, or spam; or
+  (2) enable high volume, automated, or electronic processes that send queries,
+data, or email to MarkMonitor (or its systems) or the domain name contacts (or
+its systems).
+
+MarkMonitor reserves the right to modify these terms at any time.
+
+By submitting this query, you agree to abide by this policy.
+
+MarkMonitor Domain Management(TM)
+Protecting companies and consumers in a digital world.
+
+Visit MarkMonitor at https://www.markmonitor.com
+Contact us at +1.8007459229
+In Europe, at +44.02032062220
+--

testdata/google.com/output

@@ -0,0 +1,15 @@
+
+test domain: <<<<<<<<<< google.com >>>>>>>>>>>>>>>>>>>>
+name               str               'google.com'
+tld                str               'com'
+registrar          str               'MarkMonitor, Inc.'
+registrant_country str               'US'
+creation_date      datetime.datetime 1997-09-15 09:00:00
+expiration_date    NoneType          None
+last_updated       datetime.datetime 2019-09-09 17:39:04
+status             str               'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)'
+statuses           list              ['clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)', 'clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)', 'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)', 'serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)', 'serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)', 'serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)']
+dnssec             bool              False
+name_servers       list              ['ns1.google.com', 'ns2.google.com', 'ns3.google.com', 'ns4.google.com']
+registrant         str               'Google LLC'
+emails             list              ['[email protected]']