add tests and unicode handling.

Author: jonhadfield

CLAUDE.md

@@ -53,11 +53,14 @@ This is a Python library for managing hosts files (`/etc/hosts` on Unix, `C:\Win
 - Validate IPv4/IPv6 addresses and hostnames
 - Handle comments and blank lines
 - Support for duplicate name detection with `allow_name_dupliction` parameter
+- **Unicode Support**: Full Unicode support for hostnames and comments (Python 2.7 & 3.x compatible)
+- **IDN Support**: Automatic conversion of internationalized domain names to ASCII-compatible encoding
 
 ### Module Structure
 - `python_hosts/hosts.py` - Main classes (`Hosts`, `HostsEntry`)
 - `python_hosts/utils.py` - Utility functions for validation (`is_ipv4`, `is_ipv6`, `valid_hostnames`)
 - `python_hosts/exception.py` - Custom exceptions (`HostsException`, `InvalidIPv4Address`, etc.)
+- `python_hosts/unicode_utils.py` - Unicode compatibility utilities for Python 2/3 support
 - `python_hosts/__init__.py` - Package exports
 
 ### Testing

docs/conf.py

@@ -114,7 +114,7 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 # html_theme = 'alabaster'
-html_theme = "sphinx_rtd_theme"
+html_theme = "furo"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

python_hosts/__init__.py

@@ -18,5 +18,7 @@
 from python_hosts.exception import (HostsException, HostsEntryException, # noqa: F401
                                     InvalidIPv4Address, InvalidIPv6Address,
                                     InvalidComment)
+from python_hosts.unicode_utils import (ensure_text, ensure_binary, # noqa: F401
+                                        normalize_hostname, normalize_comment)
 
 name = "python_hosts"

python_hosts/hosts.py

@@ -11,6 +11,7 @@
 of the HostsEntry class.
 """
 
+from __future__ import unicode_literals
 import sys
 
 try:
@@ -21,6 +22,9 @@
                                 dedupe_list)
 from python_hosts.exception import (InvalidIPv6Address, InvalidIPv4Address,
                                     UnableToWriteHosts)
+from python_hosts.unicode_utils import (ensure_text, ensure_binary, safe_open,
+                                        normalize_hostname, normalize_comment,
+                                        text_type, string_types)
 
 
 class HostsEntry(object):
@@ -61,10 +65,16 @@ def __init__(self,
             if not is_ipv6(address):
                 raise InvalidIPv6Address()
 
-        self.entry_type = entry_type
-        self.address = address
-        self.comment = comment
-        self.names = names
+        # Normalize all string inputs to Unicode
+        self.entry_type = ensure_text(entry_type) if entry_type else entry_type
+        self.address = ensure_text(address) if address else address
+        self.comment = normalize_comment(comment) if comment else comment
+        
+        # Normalize hostnames to Unicode and handle IDN
+        if names:
+            self.names = [normalize_hostname(name) for name in names]
+        else:
+            self.names = names
 
     def is_real_entry(self):
         return self.entry_type in ('ipv4', 'ipv6')
@@ -98,16 +108,18 @@ def get_entry_type(hosts_entry=None):
         :param hosts_entry: A line from the hosts file
         :return: 'comment' | 'blank' | 'ipv4' | 'ipv6'
         """
-        if hosts_entry and isinstance(hosts_entry, str):
+        if hosts_entry and isinstance(hosts_entry, string_types):
+            # Ensure the entry is Unicode text
+            hosts_entry = ensure_text(hosts_entry)
             entry = hosts_entry.strip()
             if not entry or not entry[0] or entry[0] == "\n":
                 return 'blank'
             if entry[0] == "#":
                 return 'comment'
             entry_chunks = entry.split()
-            if is_ipv6(entry_chunks[0]):
+            if entry_chunks and is_ipv6(entry_chunks[0]):
                 return 'ipv6'
-            if is_ipv4(entry_chunks[0]):
+            if entry_chunks and is_ipv4(entry_chunks[0]):
                 return 'ipv4'
 
     @staticmethod
@@ -117,14 +129,17 @@ def str_to_hostentry(entry):
         :param entry: A line from the hosts file
         :return: An instance of HostsEntry
         """
+        # Ensure the entry is Unicode text
+        entry = ensure_text(entry)
+        
         split_line = entry.split('#', 1)
         line = split_line[0].strip().split()
         inline_comment = split_line[1].strip() if len(split_line) == 2 else None
 
-        if is_ipv4(line[0]) and valid_hostnames(line[1:]):
+        if line and is_ipv4(line[0]) and valid_hostnames(line[1:]):
             return HostsEntry('ipv4', address=line[0], names=line[1:],
                               comment=inline_comment)
-        if is_ipv6(line[0]) and valid_hostnames(line[1:]):
+        if line and is_ipv6(line[0]) and valid_hostnames(line[1:]):
             return HostsEntry('ipv6', address=line[0], names=line[1:],
                               comment=inline_comment)
         return False
@@ -206,7 +221,7 @@ def write(self, path=None, mode='w'):
         }
         output_file_path = path if path else self.path
         try:
-            with open(output_file_path, mode) as hosts_file:
+            with safe_open(output_file_path, mode, encoding='utf-8') as hosts_file:
                 for entry in self.entries:
                     if entry.entry_type == 'comment':
                         hosts_file.write(entry.comment + "\n")
@@ -303,13 +318,19 @@ def import_url(self, url=None, force=None):
         :param url: The URL of where to download a hosts file
         :return: Counts reflecting the attempted additions
         """
-        file_contents = self.get_hosts_by_url(url=url).decode('utf-8')
+        file_contents = self.get_hosts_by_url(url=url)
+        # Handle both Python 2 and 3 URL content
+        if hasattr(file_contents, 'decode'):
+            file_contents = file_contents.decode('utf-8')
+        file_contents = ensure_text(file_contents)
         file_contents = file_contents.rstrip().replace('^M', '\n')
         file_contents = file_contents.rstrip().replace('\r\n', '\n')
         lines = file_contents.split('\n')
         skipped = 0
         import_entries = []
         for line in lines:
+            # Ensure each line is Unicode text
+            line = ensure_text(line)
             stripped_entry = line.strip()
             if (not stripped_entry) or (stripped_entry.startswith('#')):
                 skipped += 1
@@ -337,8 +358,10 @@ def import_file(self, import_file_path=None):
         invalid_count = 0
         if is_readable(import_file_path):
             import_entries = []
-            with open(import_file_path, 'r') as infile:
+            with safe_open(import_file_path, 'r', encoding='utf-8') as infile:
                 for line in infile:
+                    # Ensure line is Unicode text
+                    line = ensure_text(line)
                     stripped_entry = line.strip()
                     if (not stripped_entry) or (stripped_entry.startswith('#')):
                         skipped += 1
@@ -463,9 +486,11 @@ def populate_entries(self):
         :return: None
         """
         try:
-            with open(self.path, 'r') as hosts_file:
+            with safe_open(self.path, 'r', encoding='utf-8') as hosts_file:
                 hosts_entries = [line for line in hosts_file]
                 for hosts_entry in hosts_entries:
+                    # Ensure line is Unicode text
+                    hosts_entry = ensure_text(hosts_entry)
                     entry_type = HostsEntry.get_entry_type(hosts_entry)
                     if entry_type == "comment":
                         hosts_entry = hosts_entry.replace("\r", "")

python_hosts/unicode_utils.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+Unicode compatibility utilities for Python 2.7 and 3.x support.
+
+This module provides utilities to handle Unicode strings consistently
+across Python versions while maintaining backward compatibility.
+"""
+
+import sys
+
+# Python 2/3 compatibility
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    text_type = str
+    binary_type = bytes
+    string_types = (str,)
+    
+    def ensure_text(s, encoding='utf-8', errors='strict'):
+        """Ensure string is unicode text type."""
+        if isinstance(s, binary_type):
+            return s.decode(encoding, errors)
+        elif isinstance(s, text_type):
+            return s
+        else:
+            return text_type(s)
+    
+    def ensure_binary(s, encoding='utf-8', errors='strict'):
+        """Ensure string is binary type."""
+        if isinstance(s, text_type):
+            return s.encode(encoding, errors)
+        elif isinstance(s, binary_type):
+            return s
+        else:
+            return text_type(s).encode(encoding, errors)
+
+else:  # Python 2
+    text_type = unicode
+    binary_type = str
+    string_types = (basestring,)
+    
+    def ensure_text(s, encoding='utf-8', errors='strict'):
+        """Ensure string is unicode text type."""
+        if isinstance(s, binary_type):
+            return s.decode(encoding, errors)
+        elif isinstance(s, text_type):
+            return s
+        else:
+            return text_type(s)
+    
+    def ensure_binary(s, encoding='utf-8', errors='strict'):
+        """Ensure string is binary type."""
+        if isinstance(s, text_type):
+            return s.encode(encoding, errors)
+        elif isinstance(s, binary_type):
+            return s
+        else:
+            return binary_type(s)
+
+
+def safe_open(file_path, mode='r', encoding='utf-8', errors='strict'):
+    """
+    Open a file with proper Unicode handling for both Python 2 and 3.
+    
+    :param file_path: Path to the file
+    :param mode: File mode ('r', 'w', etc.)
+    :param encoding: Text encoding (default: utf-8)
+    :param errors: Error handling strategy
+    :return: File object
+    """
+    if PY3:
+        if 'b' not in mode:
+            return open(file_path, mode, encoding=encoding, errors=errors)
+        else:
+            return open(file_path, mode)
+    else:  # Python 2
+        import codecs
+        if 'b' not in mode:
+            return codecs.open(file_path, mode, encoding=encoding, errors=errors)
+        else:
+            return open(file_path, mode)
+
+
+def normalize_hostname(hostname):
+    """
+    Normalize hostname to ensure consistent Unicode handling.
+    
+    :param hostname: Hostname string
+    :return: Normalized Unicode hostname
+    """
+    if not hostname:
+        return hostname
+    
+    # Ensure it's Unicode text
+    hostname = ensure_text(hostname)
+    
+    # Basic normalization - lowercase and strip whitespace
+    hostname = hostname.lower().strip()
+    
+    # Handle IDN (Internationalized Domain Names)
+    try:
+        # Convert Unicode domain to ASCII-compatible encoding
+        hostname = hostname.encode('idna').decode('ascii')
+    except (UnicodeError, UnicodeDecodeError):
+        # If IDN encoding fails, keep as-is but ensure it's valid Unicode
+        pass
+    
+    return hostname
+
+
+def normalize_comment(comment):
+    """
+    Normalize comment to ensure consistent Unicode handling.
+    
+    :param comment: Comment string
+    :return: Normalized Unicode comment
+    """
+    if not comment:
+        return comment
+    
+    # Ensure it's Unicode text
+    comment = ensure_text(comment)
+    
+    # Strip whitespace but preserve internal spacing
+    comment = comment.strip()
+    
+    return comment
+
+
+def is_unicode_string(s):
+    """
+    Check if a string is a Unicode text type.
+    
+    :param s: String to check
+    :return: True if Unicode text type, False otherwise
+    """
+    return isinstance(s, text_type)
+
+
+def to_native_string(s):
+    """
+    Convert string to the native string type for the Python version.
+    
+    In Python 2: Unicode -> str (if ASCII) or keep as Unicode
+    In Python 3: Always Unicode (str)
+    
+    :param s: String to convert
+    :return: Native string type
+    """
+    if PY3:
+        return ensure_text(s)
+    else:
+        # In Python 2, try to convert to str if ASCII, otherwise keep as Unicode
+        if isinstance(s, text_type):
+            try:
+                return s.encode('ascii')
+            except UnicodeEncodeError:
+                return s
+        return s