Merge pull request #23 from jeffkala/develop

bump version

Author: jeffkala

.github/workflows/basics.yml

@@ -0,0 +1,31 @@
+---
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: "Python package"
+
+on: ["push"] # yamllint disable-line rule:truthy
+
+jobs:
+  build:
+
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: "actions/checkout@v2"
+    - name: "Set up Python ${{ matrix.python-version }}"
+      uses: "actions/setup-python@v2"
+      with:
+        python-version: "${{ matrix.python-version }}"
+    - name: "Install dependencies"
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install toml invoke poetry black
+        invoke build --no-cache
+    - name: "Run All Static Code Analysis and Tests"
+      run: |
+        invoke tests

CHANGELOG.md

@@ -1,3 +1,15 @@
+## Changelog
+
+## v0.1.2 - 2021-12-02
+
+### Added
+
+- #21 add redirect support and initial logging #21 
+
+## v0.1.1 - 2021-07-17
+
+Add linting, contributing, static analysis
+
 ## v0.1.0 - 2021-07-15
 
 Initial release

README.md

@@ -25,6 +25,18 @@ Alternatively,
 
 you can replace `python cli.py` with `pyurlcheck` on the command line.
 
+```
+▶ pyurlcheck pyurlcheck/examples/
+pyurlcheck/examples/example3.txt:4      URL Issue: https://www.ansible.com/jeff
+pyurlcheck/examples/example2.md:7       URL Issue: https://www.ansible.com/jeff
+pyurlcheck/examples/example3.md:3       URL Issue: https://www.ansible.com/jeff
+pyurlcheck/examples/example4.rst:22     URL Issue: http://google.com/france
+pyurlcheck/examples/example4.rst:23     URL Issue: http://google.com/japan
+pyurlcheck/examples/example1.md:9       URL Issue: https://www.ansible.com/jeff
+```
+
+File extensions are currently not checked; therefore all files in a directory that is passed in will be validated.
+
 ## Installation
 
 ```

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyurlcheck"
-version = "0.1.1"
+version = "0.1.2"
 description = "Search docs and validate URLs found are working properly."
 authors = ["Jeff Kala <jeff.l.kala@gmail.com>"]
 readme = "README.md"
@@ -18,6 +18,7 @@ pyurlcheck = "pyurlcheck.cli:main"
 python = "^3.6"
 click = "^7.1.2"
 requests = "^2.25.1"
+dnspython = "^2.1.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "*"

pyproject.toml

@@ -1,3 +1,3 @@
 """Pyurlcheck."""
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"

pyurlcheck/__init__.py

@@ -1,33 +1,45 @@
 """Check Whether URL is Public or Private."""
-import socket
+import logging
 import ipaddress
 from urllib.parse import urlparse
+from dns import resolver
 
 
-def _is_private(ipaddr):
-    """Check if Private RFC1918."""
-    return ipaddress.ip_address(ipaddr).is_private
+def split_url(url):
+    """Parses the URL into usable parts.
 
+    Args:
+        url (str): Exact URL found from the regex.
 
-def _get_ip(url):
-    """Take a URL and execute a name lookup to retreive IP Address Object."""
-    return socket.gethostbyname(url)
+    Returns:
+        ParseResult: Includes scheme, netloc, path, params, query, fragments.
+    """
+    url_split = urlparse(url)
+    logging.debug("URL Split: %s", url_split)
+    return url_split
 
 
-class CheckUrl:
-    """Check URL class will be used for checks that need to be done on a URL."""
+def get_ip(domain):
+    """Take a URL and execute a name lookup to retreive IP Address Object.
 
-    def __init__(self, url):
-        """Initialize URL Check."""
-        self.url = url
+    Args:
+        domain (str): URL from netloc in parse.
 
-    def split_url(self):
-        """Take Full FQDN URL and split it into usable parts."""
-        url_split = urlparse(self.url)
-        return url_split
+    Returns:
+        str: IP Address
+    """
+    result = resolver.resolve(domain, "A")
+    logging.debug("DNS Resolver: %s", result[0])
+    return result[0].address
 
-    def is_private(self):
-        """Take a URL manipulate it and check if it's Private RFC1918."""
-        split_url = self.split_url()
-        ip_addr = _get_ip(split_url)
-        return _is_private(ip_addr)
+
+def is_private(ip_addr):
+    """Take a URL manipulate it and check if it's Private RFC1918.
+
+    Args:
+        ip (str): IP address from the dns resolution.
+
+    Returns:
+        Boolean: True/False whether address is RFC1918 space.
+    """
+    return ipaddress.ip_address(ip_addr).is_private

pyurlcheck/check.py

@@ -1,37 +1,50 @@
 """Example cli using click."""
+import logging
 import sys
 
 import click
 
-from pyurlcheck.check import CheckUrl
+from pyurlcheck.check import split_url, get_ip, is_private
 from pyurlcheck.find import FindUrls
 from pyurlcheck.validate import ValidateUrl
 
 
+LOGGER = logging.getLogger(__name__)
+
+
 @click.command()
 @click.argument("input_data", type=click.Path(exists=True, file_okay=True, dir_okay=True), required=True)
-def main(input_data):
+@click.option("--log_level", type=click.Choice(["INFO", "DEBUG"], case_sensitive=False), default="INFO")
+def main(input_data, log_level):
     """Entry point into the pyurlcheck command line tool.
 
     Args:
         input_data (str): Either filename or directory to search for URLs.
+        log_level (str): Logging level to execute with. Choices ['INFO', 'DEBUG']. Default is INFO.
     """
+    logging.basicConfig(format="%(asctime)s %(message)s", stream=sys.stdout, level=getattr(logging, log_level))
     results = []
     files_urls = FindUrls(input_data).find_urls()
     for file_name, url_list in files_urls.items():
         for line_num, urls in url_list.items():
             for url in urls:
-                url_details = CheckUrl(url).split_url()
-                if url_details.scheme == "":
-                    is_valid = ValidateUrl(url, need_scheme=True).validate()
-                # print(f"URL is {url}")
-                # print(f"Is RFC1918: {is_private}")
-                else:
-                    is_valid = ValidateUrl(url).validate()
-                if not is_valid:
-                    results.append(f"{file_name}:{line_num + 1}\tURL Issue: {url}")
+                url_details = split_url(url)
+                # RFC 1918 Check, if True don't validate.
+                if not is_private(get_ip(url_details.netloc)):
+                    if url_details.scheme == "":
+                        is_valid, has_redirects = ValidateUrl(url, need_scheme=True).validate()
+                    else:
+                        is_valid, has_redirects = ValidateUrl(url).validate()
+                    if not is_valid:
+                        results.append(f"{file_name}:{line_num + 1}\tURL Issue: {url}")
+                    if has_redirects:
+                        LOGGER.info(
+                            "Redirect_Warning: %s had redirects while executing. Redirects are ' => '.join(%s)!",
+                            url,
+                            has_redirects,
+                        )
     if len(results) > 0:
-        print("\n".join(results))
+        LOGGER.info("\n".join(results))
         sys.exit(len(results))
     else:
         sys.exit(0)

pyurlcheck/cli.py

@@ -0,0 +1,7 @@
+"""Define Constants Until Config Options are Created."""
+
+IGNORE_DIRS = [
+    "__pycache__",
+    "img",
+    "images",
+]