add tld's do, com.do, mo, com.mo, cx, dz, gd, mn, gay, tl, tt; add options to makeTestdataAll

Author: rl-devops

DONE

@@ -28,3 +28,8 @@ DONE
   - allow maketestDataAll.sh -v -f -d com.sg to show the reduced input.out and the produced output of test2.py
   - add '[WHITESPACE AT END]' in input.out during input reduction test
 
+  - add tld do, com.do, mo, com.mo, cx, dz (never answeres), gd, mn, gay, tl, tt (responds with perl script)
+
+  - add -t <tld> and -d <dmian> to makeTastdataAll support a cusom domain and not meta, google default
+
+

makeTestdataAll.sh

@@ -133,7 +133,6 @@ getDnsSoaRecordAndLeaveEvidenceTldDomain()
 makeDirectoryForTld()
 {
     local tld="$1"
-    local domain="$2"
     local d="$TMPDIR/$tld"
 
     mkdir -p "$d" || {
@@ -167,23 +166,31 @@ makeTestDataOriginalOneTldDomain()
 
 domainsToTry()
 {
-    cat <<! |
+    local domain="$1"
+
+    [ "$domain" = "__DEFAULT__" ] && {
+        cat <<! |
 meta
 google
 !
-    awk  '
-    /^[ \t]*$/ { next }
-    /^[ \t]*;/ { next }
-    /^[ \t]*#/ { next }
-    { print $1 }
-    '
+        awk  '
+        /^[ \t]*$/ { next }
+        /^[ \t]*;/ { next }
+        /^[ \t]*#/ { next }
+        { print $1 }
+        '
+        return
+    }
+
+    echo "$domain"
 }
 
 makeTestDataTldFromDomains()
 {
     local tld="$1"
+    local domain="$2"
 
-    domainsToTry |
+    domainsToTry "$domain" |
     while read domain
     do
         [ "$VERBOSE" = "1" ] && echo "try: $domain.$tld"
@@ -220,12 +227,13 @@ makeRulesFromTldIfExist()
 makeTestDataOriginalOneTld()
 {
     local tld="$1"
+    local domain="$2"
 
     [ "$VERBOSE" = "1" ] && echo "try: $tld"
 
-    makeDirectoryForTld "$tld" "$domain" || exit 101
+    makeDirectoryForTld "$tld" || exit 101
     makeRulesFromTldIfExist "$tld"
-    makeTestDataTldFromDomains "$tld"
+    makeTestDataTldFromDomains "$tld" "$domain"
 }
 
 makeTestDataOriginalAllTldSupported()
@@ -245,7 +253,8 @@ $0 usage:
 -h  show the help text
 -v  switch on verbose (will show progress)
 -f  switch on force (will re analyze all)
--d  <domain> specify a domain to analize if domain == ALL andlize all tld;s
+-t  <domain> specify a tld to analize
+-a  analyze all tld currently supported
 !
     exit 0;
 }
@@ -258,7 +267,9 @@ main()
     FORCE=0
     ALL=0
 
-    while getopts "havfd:" arg;
+    local domain="__DEFAULT__"
+
+    while getopts "havft:d:" arg;
     do
         case $arg in
 
@@ -271,7 +282,11 @@ main()
         a) ALL=1
             ;;
 
-        d) domain=${OPTARG}
+        t) tld="${OPTARG}"
+            ;;
+
+        d) domain="${OPTARG}"
+            # instead of the default meta and google use this domain and combine it with the tld for processing
             ;;
 
         h | *) usage
@@ -286,7 +301,7 @@ main()
         return
     }
 
-    makeTestDataOriginalOneTld "$domain"
+    makeTestDataOriginalOneTld "$tld" "$domain"
 }
 
 main $* 2>&1 |

test2.py

@@ -155,7 +155,7 @@ def printMe(self):
         if len(self.rDict[k]):
             n = 0
             for lines in self.rDict[k]:
-                ws = " [WHITESPACE AT END] " if re.search(r'[ \t]+\r?\n', lines) else ""
+                ws = " [WHITESPACE AT END] " if re.search(r"[ \t]+\r?\n", lines) else ""
                 tab = " [TAB] " if "\t" in lines else ""  # tabs are present in this section
                 cr = " [CR] " if "\r" in lines else ""  # \r is present in this section
                 print(f"# --- {k} Section: {n} {cr}{tab}{ws}")

whois/_2_parse.py

@@ -87,7 +87,9 @@ def cleanupWhoisResponse(
         if "REDACTED FOR PRIVACY" in line:  # these lines contibute nothing so ignore
             continue
 
-        if "Please query the RDDS service of the Registrar of Record" in line: # these lines contibute nothing so ignore
+        if (
+            "Please query the RDDS service of the Registrar of Record" in line
+        ):  # these lines contibute nothing so ignore
             continue
 
         # regular responses may at the end have meta info starting with a line >>> some texte <<<

whois/__init__.py

@@ -69,6 +69,8 @@
     #
     ".com.au": "com_au",
     ".com.sg": "com_sg",
+    ".com.do": "com_do",
+    ".com.mo": "com_mo",
     #
     # TÜRKİYE (formerly Turkey)
     ".com.tr": "com_tr",

whois/tld_regexpr.py

@@ -136,9 +136,8 @@
     "updated_date": r"modified\.+:\s?(\S+)",
     "name_servers": r"nserver\.+:\s*(\S+)",
     "status": r"status\.+:\s*(\S+)",
-    "registrant": r"Holder\s+name\.+:\s*(.+)\r?\n", # not always present see meta.ax and google.ax
-    "registrant_country": r"country\.+:\s*(.+)\r?\n", # not always present see meta.ax and google.ax
-
+    "registrant": r"Holder\s+name\.+:\s*(.+)\r?\n",  # not always present see meta.ax and google.ax
+    "registrant_country": r"country\.+:\s*(.+)\r?\n",  # not always present see meta.ax and google.ax
 }
 
 aw = {
@@ -277,7 +276,7 @@
     "registrant": r"\*\* Registrant:\s+?(.+)",
     "registrant_country": None,
     "creation_date": r"Created on\.+:\s?(.+).",
-    "expiration_date": r"Expires on\.+:\s?(.+).", # note the trailing . on both dates fields
+    "expiration_date": r"Expires on\.+:\s?(.+).",  # note the trailing . on both dates fields
     "updated_date": "",
     "name_servers": r"\*\* Domain Servers:\n(?:(\S+)\n)(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)?(?:(\S+)\n)\n?",
     "status": None,
@@ -1976,3 +1975,20 @@
     "updated_date": None,
     "registrant_country": None,
 }
+
+do = {"extend": "_privateReg"}
+com_do = {"extend": "_privateReg"}
+cx = {"extend": "com"}
+dz = {"extend": "_privateReg"}
+gd = {"extend": "com"}
+mn = {"extend": "com"}
+tl = {"extend": "com"}
+gay = { "extend": "com", "_server": "whois.nic.gay" }
+tt = {"extend": "_privateReg"}
+mo = {
+    "extend": "com",
+    "creation_date": r"created on\s+(.+)",
+    "expiration_date": r"expires on\s+(.+)",
+    "name_servers": r"Domain name servers:\s*\-+(?:\s*(\S+)\n)(?:\s*(\S+)\n)?(?:\s*(\S+)\n)?(?:\s*(\S+)\n)?",
+}
+com_mo = {    "extend": "mo"}