add getopt to makeTestdataAll and improve feedback during reduce

Author: rl-devops

DONE

@@ -22,3 +22,9 @@ DONE
 
   - simplefy com.tr regex from multiple . to \.+, and add note on the trailing dot behind the date
     add meta.com.tr to regular testdata
+
+  - fix test2.py cleanpostabmble for e.g com.sg which has lines at the top with ------------
+  - add getops to makeTestDataAll.sh
+  - allow maketestDataAll.sh -v -f -d com.sg to show the reduced input.out and the produced output of test2.py
+  - add '[WHITESPACE AT END]' in input.out during input reduction test
+

makeTestdataAll.sh

@@ -2,12 +2,13 @@
 
 TMPDIR="./tmp" # the default work directory is a local tmp (exclude by .gitignore )
 
-FORCE=0 # force whois loogup by not using cached data
-VERBOSE=0 # along the way inform us on progress and results
+FORCE=0     # force whois loogup by not using cached data
+VERBOSE=0   # along the way inform us on progress and results
+ALL=0       # process all supported tld's
 
 prepTempDir()
 {
-    # make the work dir if it does nt exist
+    # make the work dir if it does not exist
     mkdir -p "$TMPDIR" || {
         echo "FATAL: cannot make test dir: $TMPDIR" >&2
         exit 101
@@ -27,23 +28,27 @@ testNameserverExistsInInputAndOutput()
     local dns="$d/__dns-ns"
 
     [ -s "$dns" ] || return # dont bother at all if we have no dns file
+    [ -s "$d/input" ] || return # dont bother if we have no input file
 
     rm -f "$d/error.ns"
 
     cat "$dns" |
     awk '{ print $NF }' |
     while read ns
     do
-        [ -s "$d/input" ] || return # dont bother if we have no input file
-
+        [ "$VERBOSE" = "1" ] && echo "test $ns in input"
         grep -q -i "$ns" "$d/input" && {
             # only test in the output if it is present in the input
+
+            [ "$VERBOSE" = "1" ] && echo "test $ns in output"
             grep -q -i "$ns" "$d/output" || {
                 echo "ERROR: output;  missing nameserver '$ns' for tld: $tld" |
                 tee -a "$d/error.ns"
+                return 1
             }
         }
     done
+    return 0
 }
 
 cleanupTldTestDirectory()
@@ -153,15 +158,10 @@ makeTestDataOriginalOneTldDomain()
 
     getDnsSoaRecordAndLeaveEvidenceTldDomain "$tld" "$domain" || return 1
 
-    # what domain did we test
-    touch "$d/__domain__$zz"
-
-    # store the nameservers from dns
-    host -t ns "$zz" > "$d/__dns-ns__$zz"
-
+    touch "$d/__domain__$zz"                # what domain did we test
+    host -t ns "$zz" > "$d/__dns-ns__$zz"   # store the nameservers from dns
     getTestDataInputForTldAndDomain "$tld" "$domain" || return 1
-
-    getTestDataOutputForTldAndDomain "$tld" "$domain"
+    getTestDataOutputForTldAndDomain "$tld" "$domain" || return 1
     return 0
 }
 
@@ -188,13 +188,24 @@ makeTestDataTldFromDomains()
     do
         [ "$VERBOSE" = "1" ] && echo "try: $domain.$tld"
 
-        makeTestDataOriginalOneTldDomain "$tld" "$domain"
-
-        [ -s "$TMPDIR/$tld/input" ] && {
-            [ "$VERBOSE" = "1" ] && ls -l "$TMPDIR/$tld/"
-            testNameserverExistsInInputAndOutput "$tld" && break
+        makeTestDataOriginalOneTldDomain "$tld" "$domain" && {
+            [ -s "$TMPDIR/$tld/input" ] && {
+                [ "$VERBOSE" = "1" ] && {
+                    ls -l "$TMPDIR/$tld/"
+                }
+            }
+            break
         }
     done
+
+    [ "$VERBOSE" = "1" ] && {
+        # show the rediced input and the output
+        cat "$TMPDIR/$tld/input.out" "$TMPDIR/$tld/output"
+    }
+
+    testNameserverExistsInInputAndOutput "$tld" && {
+        return
+    }
 }
 
 makeRulesFromTldIfExist()
@@ -227,19 +238,55 @@ makeTestDataOriginalAllTldSupported()
     done
 }
 
+usage()
+{
+    cat <<!
+$0 usage:
+-h  show the help text
+-v  switch on verbose (will show progress)
+-f  switch on force (will re analyze all)
+-d  <domain> specify a domain to analize if domain == ALL andlize all tld;s
+!
+    exit 0;
+}
+
 main()
 {
+    [ $# -eq 0 ] && usage
+
+    VERBOSE=0
+    FORCE=0
+    ALL=0
+
+    while getopts "havfd:" arg;
+    do
+        case $arg in
+
+        v) VERBOSE=1
+            ;;
+
+        f) FORCE=1
+            ;;
+
+        a) ALL=1
+            ;;
+
+        d) domain=${OPTARG}
+            ;;
+
+        h | *) usage
+            ;;
+        esac
+    done
+
     prepTempDir
 
-    [ "$#" = "0" ] && {
+    [ "$ALL" = "1" ] && {
         makeTestDataOriginalAllTldSupported
         return
     }
 
-    for tld in $*
-    do
-        makeTestDataOriginalOneTld "$tld"
-    done
+    makeTestDataOriginalOneTld "$domain"
 }
 
 main $* 2>&1 |

test2.py

@@ -120,7 +120,7 @@ def cleanupWhoisResponse(
                     percentSeen = True
 
             if postambleSeen is False:
-                if line.startswith("--") or line.startswith(">>> ") or line.startswith("Copyright notice"):
+                if line.startswith("-- ") or line.startswith(">>> ") or line.startswith("Copyright notice"):
                     postambleSeen = True
 
             if postambleSeen is True:
@@ -155,9 +155,10 @@ def printMe(self):
         if len(self.rDict[k]):
             n = 0
             for lines in self.rDict[k]:
-                tab = " [TAB] " if "\t" in lines else "-------"  # tabs are present in this section
-                cr = " [CR] " if "\r" in lines else "------"  # \r is present in this section
-                print(f"# ------------- {k} Section: {n} {cr}{tab}---------")
+                ws = " [WHITESPACE AT END] " if re.search(r'[ \t]+\r?\n', lines) else ""
+                tab = " [TAB] " if "\t" in lines else ""  # tabs are present in this section
+                cr = " [CR] " if "\r" in lines else ""  # \r is present in this section
+                print(f"# --- {k} Section: {n} {cr}{tab}{ws}")
                 n += 1
                 print(lines)