Merge pull request #330 from maarten-boot/master

sync with 1.20230829.1 whoisdomain; add makefile to test whl

Author: maarten-boot

makefile

@@ -0,0 +1,16 @@
+
+
+all: reformat test build testwhl
+
+reformat:
+	./reformat-code.sh
+
+test:
+	./test2.py -d google.com
+
+build:
+	python -m build
+	tar tvf dist/*.tar.gz
+
+testwhl:
+	./testwhl.sh 2>2 | tee 1

setup.py

@@ -10,7 +10,20 @@
     download_url="https://github.com/DannyCork/python-whois/releases/latest",
     url="https://github.com/DannyCork/python-whois/",
     platforms=["any"],
-    packages=["whois"],
+    package_dir={
+        "whois": "whois",
+        "whois.tldDb": "whois/tldDb",
+        "whois.context": "whois/context",
+        "whois.cache": "whois/cache",
+        "whois.strings": "whois/strings",
+    },
+    packages=[
+        "whois",
+        "whois.tldDb",
+        "whois.context",
+        "whois.cache",
+        "whois.strings",
+    ],
     keywords=[
         "Python",
         "whois",

testwhl.py

@@ -0,0 +1,5 @@
+import whois
+
+domain = "google.com"
+d = whois.query(domain, verbose=True)
+print(d.__dict__)

testwhl.sh

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+ENV="tmp/t1"
+
+mkdir -p tmp
+rm -rf tmp/t1
+python3 -m venv ${ENV}
+source ./${ENV}/bin/activate
+pip3 install dist/*.whl
+
+python testwhl.py

whois/_0_init_tld.py

@@ -1,20 +1,18 @@
 import re
-import sys
 
 from typing import (
     Dict,
     List,
     Any,
 )
 
-from .tld_regexpr import ZZ
 from .exceptions import (
     UnknownTld,
 )
 
-# from .parameterContext import ParameterContext # not needed yet here
+from .tldDb.tld_regexpr import ZZ
+
 
-Verbose: bool = False
 TLD_RE: Dict[str, Dict[str, Any]] = {}
 REG_COLLECTION_BY_KEY: Dict[str, Any] = {}
 
@@ -44,12 +42,10 @@ def _get_tld_re(
     if "extend" in tmp:
         del tmp["extend"]
 
-    # we want now to exclude _server hints
+    # we want now to exclude andy key starting with _ like _server,_test, ...
+    # dont recompile each re by themselves, reuse existing compiled re
     tld_re = dict(
-        # (k, re.compile(v, re.IGNORECASE) if (isinstance(v, str) and k[0] != "_") else v) for k, v in tmp.items()
-        # dont recompile each re by themselves, reuse existing compiled re
-        (k, REG_COLLECTION_BY_KEY[k][v] if (isinstance(v, str) and k[0] != "_") else v)
-        for k, v in tmp.items()
+        (k, REG_COLLECTION_BY_KEY[k][v] if (isinstance(v, str) and k[0] != "_") else v) for k, v in tmp.items()
     )
 
     # meta domains start with _: examples _centralnic and _donuts
@@ -77,8 +73,6 @@ def _initOne(
 
     # also automatically see if we can internationalize the domains to the official ascii string
     TLD_RE[tld2] = what
-    if Verbose:
-        print(f"{tld} -> {tld2}", file=sys.stderr)
 
 
 def _buildRegCollection(
@@ -141,20 +135,13 @@ def filterTldToSupportedPattern(
     # we have max 2 levels so first check if the last 2 are in our list
     tld = f"{d[-2]}.{d[-1]}"
     if tld in ZZ:
-        if verbose:
-            print(f"we have {tld}", file=sys.stderr)
         return tld
 
     # if not check if the last item  we have
     tld = f"{d[-1]}"
     if tld in ZZ:
-        if verbose:
-            print(f"we have {tld}", file=sys.stderr)
         return tld
 
-    if verbose:
-        print(f"we DONT have {tld}", file=sys.stderr)
-
     # if not fail
     a = f"The TLD {tld} is currently not supported by this package."
     b = "Use validTlds() to see what toplevel domains are supported."
@@ -169,7 +156,7 @@ def mergeExternalDictWithRegex(
     for tld in aDict.keys():
         ZZ[tld] = aDict[tld]
 
-    # reprocess te regexes we newly defined or overrode
+    # reprocess the regexes we newly defined or overrode
     override = True
     for tld in aDict.keys():
         _initOne(tld, override)

whois/__init__.py

@@ -11,6 +11,7 @@
     Callable,
 )
 
+
 from .version import (
     VERSION,
 )
@@ -32,27 +33,44 @@
     mergeExternalDictWithRegex,
 )
 
-from .doQuery import query, q2
-from .lastWhois import get_last_raw_whois_data
+from .processWhoisDomainRequest import ProcessWhoisDomainRequest
+from .lastWhois import (
+    get_last_raw_whois_data,
+    initLastWhois,
+)
 from .doWhoisCommand import setMyCache
-from .noneStrings import NoneStrings, NoneStringsAdd
-from .quotaStrings import QuotaStrings, QuotaStringsAdd
+from .domain import Domain
 
+from .strings.noneStrings import NoneStrings, NoneStringsAdd
+from .strings.quotaStrings import QuotaStrings, QuotaStringsAdd
+from .tldDb.tld_regexpr import ZZ
+from .context.dataContext import DataContext
+from .context.parameterContext import ParameterContext
+from .cache.simpleCacheBase import SimpleCacheBase
+from .cache.simpleCacheWithFile import SimpleCacheWithFile
+from .cache.dummyCache import DummyCache
+from .cache.dbmCache import DBMCache
 
-from .doParse import (
-    cleanupWhoisResponse,
-)
+HAS_REDIS = False
+try:
+    import redis
 
-from .tld_regexpr import ZZ
-from .domain import Domain
-from .parameterContext import ParameterContext
+    HAS_REDIS = True
+except Exception as e:
+    _ = e
+
+if HAS_REDIS:
+    from .cache.redisCache import RedisCache
 
-from .simpleCacheBase import SimpleCacheBase
-from .simpleCacheWithFile import SimpleCacheWithFile
-from .dummyCache import DummyCache
-from .dbmCache import DBMCache
+WHOISDOMAIN: str = ""
+if os.getenv("WHOISDOMAIN"):
+    WHOISDOMAIN = str(os.getenv("WHOISDOMAIN"))
 
-# from .redisCache import RedisCache
+WD = WHOISDOMAIN.upper().split(":")
+
+SIMPLISTIC = False
+if "SIMPLISTIC" in WD:
+    SIMPLISTIC = True
 
 TLD_LIB_PRESENT: bool = False
 try:
@@ -74,7 +92,6 @@
     # from init_tld
     "validTlds",
     "TLD_RE",
-    "mergeExternalDictWithRegex",
     # from version
     "VERSION",
     # from parameterContext
@@ -112,6 +129,77 @@ def _inner(*args: str, **kw: Any) -> Dict[str, Any]:
     return _inner
 
 
+def q2(
+    domain: str,
+    pc: ParameterContext,
+) -> Optional[Domain]:
+
+    initLastWhois()
+
+    dc = DataContext(
+        pc=pc,
+        domain=domain,
+    )
+    pwdr = ProcessWhoisDomainRequest(
+        domain=domain,
+        pc=pc,
+        dc=dc,
+    )
+
+    return pwdr.processRequest()
+
+
+def query(
+    domain: str,
+    force: bool = False,
+    cache_file: Optional[str] = None,
+    cache_age: int = 60 * 60 * 48,
+    slow_down: int = 0,
+    ignore_returncode: bool = False,
+    server: Optional[str] = None,
+    verbose: bool = False,
+    with_cleanup_results: bool = False,
+    internationalized: bool = False,
+    include_raw_whois_text: bool = False,
+    return_raw_text_for_unsupported_tld: bool = False,
+    timeout: Optional[float] = None,
+    parse_partial_response: bool = False,
+    cmd: str = "whois",
+    simplistic: bool = False,
+    withRedacted: bool = False,
+    pc: Optional[ParameterContext] = None,
+    # if you use pc as argument all above params (except domain are ignored)
+) -> Optional[Domain]:
+    # see documentation about paramaters in parameterContext.py
+
+    assert isinstance(domain, str), Exception("`domain` - must be <str>")
+
+    if pc is None:
+        pc = ParameterContext(
+            force=force,
+            cache_file=cache_file,
+            cache_age=cache_age,
+            slow_down=slow_down,
+            ignore_returncode=ignore_returncode,
+            server=server,
+            verbose=verbose,
+            with_cleanup_results=with_cleanup_results,
+            internationalized=internationalized,
+            include_raw_whois_text=include_raw_whois_text,
+            return_raw_text_for_unsupported_tld=return_raw_text_for_unsupported_tld,
+            timeout=timeout,
+            parse_partial_response=parse_partial_response,
+            cmd=cmd,
+            simplistic=simplistic,
+            withRedacted=withRedacted,
+        )
+
+    if verbose:
+        print(pc, file=sys.stderr)
+
+    return q2(domain=domain, pc=pc)
+
+
 # Add get function to support return result in dictionary form
 get = _result2dict(query)
 
@@ -129,3 +217,57 @@ def getTestHint(tldString: str) -> Optional[str]:
         return None
 
     return str(ZZ[tldString][k])
+
+
+def cleanupWhoisResponse(
+    whoisStr: str,
+    verbose: bool = False,
+    with_cleanup_results: bool = False,
+    withRedacted: bool = False,
+    pc: Optional[ParameterContext] = None,
+) -> str:
+    tmp2: List[str] = []
+
+    if pc is None:
+        pc = ParameterContext(
+            verbose=verbose,
+            withRedacted=withRedacted,
+            with_cleanup_results=with_cleanup_results,
+        )
+
+    skipFromHere = False
+    tmp: List[str] = whoisStr.split("\n")
+    for line in tmp:
+        if skipFromHere is True:
+            continue
+
+        # some servers respond with: % Quota exceeded in the comment section (lines starting with %)
+        if "quota exceeded" in line.lower():
+            raise WhoisQuotaExceeded(whoisStr)
+
+        if pc.with_cleanup_results is True and line.startswith("%"):  # only remove if requested
+            continue
+
+        if pc.withRedacted is False:
+            if "REDACTED FOR PRIVACY" in line:  # these lines contibute nothing so ignore
+                continue
+
+        if (
+            "Please query the RDDS service of the Registrar of Record" in line
+        ):  # these lines contibute nothing so ignore
+            continue
+
+        # regular responses may at the end have meta info starting with a line >>> some texte <<<
+        # similar trailing info exists with lines starting with -- but we wil handle them later
+        # unfortunalery we have domains (google.st) that have this early at the top
+        if 0:
+            if line.startswith(">>>"):
+                skipFromHere = True
+                continue
+
+        if line.startswith("Terms of Use:"):  # these lines contibute nothing so ignore
+            continue
+
+        tmp2.append(line.strip("\r"))
+
+    return "\n".join(tmp2)