1.0.4.dev2 (Refactoring and test enhancements) (#73)

Author: mikeqfu

MANIFEST.in

@@ -1,9 +1,20 @@
 # Include package data
 include LICENSE
+include README.md
+include CHANGELOG.md
+include CITATION.cff
+
+# Include package data in pyrcs/data
 recursive-include pyrcs/data *
 
-# Exclude tests
+# Exclude development and build directories
 prune tests
+prune docs
 prune tutorials
+prune .venv
 prune venv
 prune dist
+prune *.egg-info
+
+# Global exclusions (optional but good)
+global-exclude __pycache__

docs/source/utils.rst

@@ -16,12 +16,12 @@ Validate inputs
     :toctree: _generated/
     :template: function.rst
 
-    is_home_connectable
+    is_homepage_connectable
     is_str_float
     validate_initial
     validate_page_name
-    collect_in_fetch_verbose
-    fetch_all_verbose
+    get_collect_verbosity_for_fetch
+    get_batch_fetch_verbosity
 
 Print messages
 ~~~~~~~~~~~~~~
@@ -31,10 +31,10 @@ Print messages
     :template: function.rst
 
     format_confirmation_prompt
-    print_collect_msg
-    print_conn_err
-    print_inst_conn_err
-    print_void_msg
+    print_collection_message
+    print_connection_warning
+    print_instance_connection_error
+    print_void_collection_message
 
 Save and retrieve pre-packed data
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pyproject.toml

@@ -44,7 +44,7 @@ keywords = [
 ]
 requires-python = ">=3.12"
 dependencies = [
-    "pyhelpers >= 2.3.1",
+    "pyhelpers >= 2.3.2",
     "beautifulsoup4"
 ]
 classifiers = [

pyrcs/_base.py

@@ -16,8 +16,9 @@
 from pyhelpers.store import load_data, save_data
 
 from .parser import get_catalogue, get_introduction, get_last_updated_date
-from .utils import cd_data, collect_in_fetch_verbose, format_confirmation_prompt, home_page_url, \
-    print_collect_msg, print_conn_err, print_inst_conn_err, print_void_msg
+from .utils import cd_data, format_confirmation_prompt, get_collect_verbosity_for_fetch, \
+    homepage_url, print_collection_message, print_connection_warning, \
+    print_instance_connection_error, print_void_collection_message
 
 
 class _Base:
@@ -30,7 +31,7 @@ class _Base:
     #: The key for accessing the data.
     KEY: str = ''
     #: The URL of the main web page for the data.
-    URL: str = home_page_url()
+    URL: str = homepage_url()
     #: The key used to reference the last updated date in the data.
     KEY_TO_LAST_UPDATED_DATE: str = 'Last updated date'
 
@@ -72,7 +73,7 @@ def __init__(self, data_dir=None, content_type=None, data_category="", data_clus
             'http://www.railwaycodes.org.uk/'
         """
 
-        print_conn_err(verbose=verbose)
+        print_connection_warning(verbose=verbose)
 
         self.catalogue, self.introduction = None, None
 
@@ -174,6 +175,7 @@ def _cdd(self, *sub_dir, mkdir=True, **kwargs):
 
     @staticmethod
     def _format_confirmation_message(data_name, confirmation_prompt=None, initial=None, **kwargs):
+        # noinspection PyShadowingNames
         """
         Generates a confirmation prompt message.
 
@@ -185,6 +187,25 @@ def _format_confirmation_message(data_name, confirmation_prompt=None, initial=No
         :type data_name: str
         :return: The generated confirmation prompt as a string.
         :rtype: str
+
+        **Examples**::
+
+            >>> from pyrcs._base import _Base
+            >>> from pyrcs.utils import format_confirmation_prompt
+            >>> _b = _Base()
+            >>> data_name = '"test_data_name"'
+            >>> prompt = _b._format_confirmation_message(data_name)
+            >>> prompt
+            'To collect data of "test_data_name"\n?'
+            >>> prompt = _b._format_confirmation_message(data_name, format_confirmation_prompt)
+            >>> prompt
+            'To collect data of "test_data_name"\n?'
+            >>> prompt = _b._format_confirmation_message(data_name, "test message")
+            >>> prompt
+            'test message'
+            >>> prompt = _b._format_confirmation_message(data_name, format_confirmation_prompt, 'a')
+            >>> prompt
+            'To collect data of "test_data_name" beginning with "a"\n?'
         """
 
         if confirmation_prompt:
@@ -224,80 +245,121 @@ def _collect_data_from_source(self, data_name, method, url=None, initial=None,
                                   confirmation_prompt=None, verbose=False, raise_error=False,
                                   **kwargs):
         """
-        Collects data from the specified source webpage(s).
+        Collects and parses data from a specified source webpage.
 
-        :param data_name: Name of the data to be collected.
-        :type data_name: str
-        :param url: URL of the webpage from which the data will be collected.
-        :type url: str
-        :param method: A callable function or method used to parse and extract data from the
-            webpage. This function should accept the following parameters:
+        :param data_name: The descriptive name of the data being collected
+            (used for lookups and messages).
+        :type data_name: str | None
+        :param method: The parsing function to execute upon successful data retrieval.
+            The function **must** accept:
+
+            - ``source`` (*requests.Response*): The HTTP response object.
+            - ``verbose`` (*bool | int*): The verbosity flag.
 
-            - ``source``: The response object from the HTTP request.
-            - ``verbose``: Whether to print additional information during extraction.
+            The function **may** optionally accept:
+
+            - ``data_name`` (*str*): Injected automatically if present in the function signature.
+            - ``initial`` (*str*): Injected automatically if present in the function signature.
+            - Any other arguments passed via ``**kwargs``.
 
         :type method: typing.Callable
-        :param initial: The initial letter of the desired code or data; defaults to ``None``.
+        :param url: The target URL. If ``None``, the method attempts to retrieve the URL from
+            ``self.catalogue`` using ``initial`` or ``data_name`` as the key.
+        :type url: str | None
+        :param initial: The initial letter/code used to categorize the data
+            (e.g. 'A' for stations starting with A); it is used as a fallback key for URL lookup
+            if ``url`` is not provided.
         :type initial: str | None
-        :param additional_fields: Extra key-value pairs to be included in the returned dictionary
-            if data collection fails or is canceled; defaults to ``None``.
+        :param additional_fields: Key-value pairs to include in the fallback dictionary
+            if data collection fails; it is useful for ensuring consistent data structure
+            (e.g. returning ``None`` for specific columns).
         :type additional_fields: dict | str | None
         :param confirmation_required: Whether user confirmation is required;
             if ``confirmation_required=True`` (default), prompts the user for confirmation
             before proceeding with data collection.
         :type confirmation_required: bool
-        :param confirmation_prompt:
-        :type confirmation_prompt:
-        :param verbose: Whether to print relevant information in the console; defaults to ``False``.
+        :param confirmation_prompt: A custom message or a callable that generates a message
+            for the confirmation prompt.
+        :type confirmation_prompt: str | typing.Callable | None
+        :param verbose: Whether to print status messages and errors to the console;
+            defaults to ``False``.
         :type verbose: bool | int
-        :param raise_error: Whether to raise the provided exception;
-            if ``raise_error=False`` (default), the error will be suppressed.
+        :param raise_error: If ``True``, exceptions (network or parsing) are raised to the caller;
+            if ``False`` (default), exceptions are caught and fallback data is returned.
         :type raise_error: bool
-        :return: The collected data.
+        :param kwargs: [Optional] Additional keyword arguments passed directly to the ``method``.
+        :return: The data returned by ``method``, or a dictionary containing fallback values
+            (e.g. ``{key: None}``) on failure.
         :rtype: pandas.DataFrame | dict | None
+        :raises ValueError: If the URL cannot be resolved and ``raise_error=True``.
+        :raises requests.RequestException: If a network error occurs and ``raise_error=True``.
+
+        **Examples**::
+
+            >>> from pyrcs._base import _Base
+            >>> _b = _Base()
+            >>> _b.catalogue = {'A': 'https://github.com/mikeqfu/pyrcs'}
+            >>> _b._collect_data_from_source("test_data_name", method=_b._fallback_data)
         """
 
+        # Confirmation step
         prompt = self._format_confirmation_message(
             data_name=data_name, confirmation_prompt=confirmation_prompt, initial=initial)
 
         if not confirmed(prompt=prompt, confirmation_required=confirmation_required):
             return None
 
-        print_collect_msg(
+        print_collection_message(
             data_name=data_name, initial=initial, verbose=verbose,
             confirmation_required=confirmation_required)
 
+        # Prepare fallback data
         fallback_data = self._fallback_data(key=initial, additional_fields=additional_fields)
 
-        url_ = copy.copy(url or self.catalogue.get(initial or data_name))
-        if not url_:
-            if initial and verbose:
-                print(f'No data is available for codes beginning with "{initial}".')
-            elif data_name and not url:
-                print('Key not found in `.catalogue`. '
-                      'Check `.catalogue` for valid keys, and verify `initial` or `data_name`.')
+        # Resolve URL
+        target_url = url or self.catalogue.get(initial or data_name)
+
+        if not target_url:
+            if initial:
+                err_msg = f'No data is available for codes beginning with "{initial}".'
+            else:  # defaults to data_name context
+                err_msg = \
+                    f'"{data_name}" not found in `.catalogue`. Check `.catalogue` for valid keys.'
+
+            if raise_error:
+                raise ValueError(err_msg)
+            elif verbose:
+                print(err_msg)
             return fallback_data
 
+        # Fetch and process
         try:
-            source = requests.get(url=url_, headers=fake_requests_headers())
-            source.raise_for_status()  # Raises HTTPError for bad responses (4xx, 5xx)
-        except Exception as e:
-            print_inst_conn_err(verbose=verbose, e=e)
-            return fallback_data
+            # Network request
+            source = requests.get(url=target_url, headers=fake_requests_headers(), timeout=30)
+            source.raise_for_status()  # Raises HTTPError for bad responses
 
-        # Build kwargs dynamically based on method signature
-        kwargs.update({'source': source, 'verbose': verbose})
+            # Dynamic argument injection
+            collector_kwargs = kwargs.copy()
+            collector_kwargs.update({'source': source, 'verbose': verbose})
 
-        for param in ('data_name', 'initial'):
-            if param in inspect.signature(method).parameters:
-                kwargs.update({param: locals()[param]})
+            # Inspect the collector method to see if it requires extra arguments
+            params = inspect.signature(method).parameters
+            if 'data_name' in params:
+                collector_kwargs['data_name'] = data_name
+            if 'initial' in params:
+                collector_kwargs['initial'] = initial
 
-        # Attempt method execution
-        try:
-            return method(**kwargs)
-        except Exception as e:
-            _print_failure_message(
-                e, prefix="Failed. Error:", verbose=verbose, raise_error=raise_error)
+            # Execute Parsing Method
+            data = method(**collector_kwargs)
+
+            return data
+
+        except requests.RequestException as e:  # Handle network/HTTP errors
+            print_instance_connection_error(verbose=verbose, e=e, raise_error=raise_error)
+            return fallback_data
+
+        except Exception as e:  # Handle parsing/method errors
+            _print_failure_message(e, "Failed. Error:", verbose=verbose, raise_error=raise_error)
             return fallback_data
 
     def _make_file_pathname(self, data_name, ext=".pkl", data_dir=None, sub_dir=None, **kwargs):
@@ -410,7 +472,7 @@ def _save_data_to_file(self, data, data_name, ext=".pkl", dump_dir=None, sub_dir
             save_data(data=data, path_to_file=path_to_file, verbose=(verbose == 2), **kwargs)
 
         else:
-            print_void_msg(data_name=data_name, verbose=verbose)
+            print_void_collection_message(data_name=data_name, verbose=verbose)
 
     def _fetch_data_from_file(self, data_name, method, ext=".pkl", update=False, dump_dir=None,
                               verbose=False, raise_error=False, data_dir=None, sub_dir=None,
@@ -485,7 +547,7 @@ def _fetch_data_from_file(self, data_name, method, ext=".pkl", update=False, dum
                 data = load_data(path_to_file, verbose=(verbose == 2))
 
             else:
-                verbose_ = collect_in_fetch_verbose(data_dir=dump_dir, verbose=verbose)
+                verbose_ = get_collect_verbosity_for_fetch(data_dir=dump_dir, verbose=verbose)
 
                 kwargs.update({'confirmation_required': False, 'verbose': verbose_})
 

pyrcs/_updater.py

@@ -6,10 +6,11 @@
 
 from .collector import LineData, OtherAssets
 from .parser import get_site_map
-from .utils import is_home_connectable, print_conn_err
+from .utils import is_homepage_connectable, print_connection_warning
 
 
 def _update_prepacked_data(verbose=False, interval=5, **kwargs):
+    # noinspection PyUnresolvedReferences
     """
     Updates pre-packed data.
 
@@ -24,8 +25,8 @@ def _update_prepacked_data(verbose=False, interval=5, **kwargs):
         >>> _update_prepacked_data(verbose=True)
     """
 
-    if not is_home_connectable():
-        print_conn_err(verbose=verbose)
+    if not is_homepage_connectable():
+        print_connection_warning(verbose=verbose)
         print("Unable to update the data.")
 
     else:

pyrcs/collector.py

@@ -17,17 +17,17 @@
 from .other_assets import Buzzer, Depots, Features, HabdWild, SignalBoxes, Stations, Telegraph, \
     Tunnels, Viaducts, WaterTroughs
 from .parser import get_category_menu
-from .utils import is_home_connectable, print_conn_err, print_inst_conn_err
+from .utils import is_homepage_connectable, print_connection_warning, print_instance_connection_error
 
 
 class _Base:
     #: The name of the data.
     NAME: str = 'Railway Codes and other data'
 
     def __init__(self, update=False, verbose=True, raise_error=False):
-        if not is_home_connectable():
+        if not is_homepage_connectable():
             self.connected = False
-            print_conn_err(verbose=verbose)
+            print_connection_warning(verbose=verbose)
 
         else:
             self.connected = True
@@ -150,7 +150,7 @@ def update(self, confirmation_required=True, verbose=False, interval=5, init_upd
         """
 
         if not self.connected:
-            print_inst_conn_err(verbose=verbose)
+            print_instance_connection_error(verbose=verbose)
 
         else:
             if confirmed("To update line data\n?", confirmation_required=confirmation_required):
@@ -320,7 +320,7 @@ def update(self, confirmation_required=True, verbose=False, interval=5, init_upd
         """
 
         if not self.connected:
-            print_inst_conn_err(verbose=verbose)
+            print_instance_connection_error(verbose=verbose)
 
         else:
             if confirmed("To update data of other assets\n?", confirmation_required):

pyrcs/data/.metadata

@@ -5,7 +5,7 @@
     "Author": "Qian Fu",
     "Affiliation": "University of Birmingham",
     "Email": "q.fu@bham.ac.uk",
-    "Version": "1.0.4.dev1",
+    "Version": "1.0.4.dev2",
     "License": "MIT",
     "First release": "August 2019"
 }

pyrcs/line_data/bridge.py

@@ -11,7 +11,7 @@
 
 from .._base import _Base
 from ..parser import _get_last_updated_date
-from ..utils import home_page_url
+from ..utils import homepage_url
 
 
 class Bridges(_Base):
@@ -25,7 +25,7 @@ class Bridges(_Base):
     #: The key for accessing the data.
     KEY: str = 'Bridges'
     #: The URL of the main webpage for the data.
-    URL: str = urllib.parse.urljoin(home_page_url(), '/bridges/bridges0.shtm')
+    URL: str = urllib.parse.urljoin(homepage_url(), '/bridges/bridges0.shtm')
     #: The key used to reference the last updated date in the data.
     KEY_TO_LAST_UPDATED_DATE: str = 'Last updated date'