Updated gitignore

Author: bram

pyproject.toml

@@ -22,17 +22,17 @@ requires-python = ">=3.9"
 license = {text = "MIT"}
 dependencies = [
     "polib==1.2.0",
-    "openai==1.58.1",
-    "python-dotenv==1.0.0",
-    "pytest==8.2.2",
+    "openai==1.99.9",
+    "python-dotenv==1.0.1",
+    "pytest==8.3.4",
     "tenacity==9.0.0",
     "setuptools-scm==8.1.0",
     "pycountry==24.6.1",
-    "anthropic==0.48.0",
+    "anthropic==0.63.0",
     "requests==2.32.3",
-    "responses==0.25.6",
-    "isort==6.0.1",
-    "tomli>=1.2.0; python_version<'3.11'",
+    "responses==0.25.8",
+    "isort==5.13.2",
+    "tomli==2.2.1",
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",

python_gpt_po/main.py

@@ -199,9 +199,11 @@ def main():
                 languages = parse_languages(args.lang)
                 logging.info("Using specified languages: %s", ', '.join(languages))
             else:
+                respect_gitignore = not args.no_gitignore  # Invert the flag
                 languages = LanguageDetector.detect_languages_from_folder(
                     args.folder,
-                    use_folder_structure=args.folder_language
+                    use_folder_structure=args.folder_language,
+                    respect_gitignore=respect_gitignore
                 )
                 detection_method = "folder structure" if args.folder_language else "metadata"
                 logging.info("Auto-detected languages from %s: %s", detection_method, ', '.join(languages))

python_gpt_po/models/config.py

@@ -2,10 +2,14 @@
 Configuration classes for the PO translator application.
 """
 
+from __future__ import annotations
 from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
 from .enums import ModelProvider
-from .provider_clients import ProviderClients
+
+if TYPE_CHECKING:
+    from .provider_clients import ProviderClients
 
 
 @dataclass

python_gpt_po/services/language_detector.py

@@ -4,23 +4,76 @@
 
 import logging
 import os
-import re
-from typing import List
+from typing import List, Optional, Set
 
 import polib
+import pycountry
+
+from ..utils.gitignore import create_gitignore_parser
 
 
 class LanguageDetector:
     """Detects languages from PO files in a directory."""
 
+    # Cache for valid language codes
+    _valid_codes_cache: Optional[Set[str]] = None
+
+    @classmethod
+    def _get_valid_language_codes(cls) -> Set[str]:
+        """Get a set of all valid language codes using pycountry and known Django codes."""
+        if cls._valid_codes_cache is not None:
+            return cls._valid_codes_cache
+
+        valid_codes = set()
+
+        # Add ISO 639-1 codes (2-letter codes like 'en', 'fr')
+        for lang in pycountry.languages:
+            if hasattr(lang, 'alpha_2'):
+                valid_codes.add(lang.alpha_2)
+            if hasattr(lang, 'alpha_3'):
+                # Also add 3-letter codes
+                valid_codes.add(lang.alpha_3)
+
+        # Add locale codes with country (like 'en_US', 'pt_BR')
+        for country in pycountry.countries:
+            if hasattr(country, 'alpha_2'):
+                country_code = country.alpha_2
+                # Common combinations
+                common_langs = [
+                    'en', 'es', 'fr', 'de', 'pt', 'zh', 'ar', 'ru', 'ja', 'ko',
+                    'it', 'nl', 'pl', 'tr', 'sv', 'da', 'no', 'fi', 'uk', 'cs',
+                    'hu', 'ro', 'bg', 'hr', 'sr', 'sl', 'sk', 'lt', 'lv', 'et',
+                    'hi', 'ur', 'bn', 'ta', 'te', 'mr', 'gu', 'kn', 'ml', 'pa', 'ne'
+                ]
+                for lang_code in common_langs:
+                    valid_codes.add(f"{lang_code}_{country_code}")
+                    valid_codes.add(f"{lang_code}-{country_code}")
+
+        # Add Django special codes
+        django_special = {
+            'zh_Hans', 'zh_Hant', 'zh-hans', 'zh-hant',  # Chinese variants
+            'sr_Latn', 'sr-latn', 'sr@latin',  # Serbian variants
+            'be@tarask',  # Belarusian variant
+            'en-us', 'en-gb', 'en-au', 'en-ca',  # English variants
+            'es-es', 'es-mx', 'es-ar',  # Spanish variants
+            'pt-pt', 'pt-br',  # Portuguese variants
+            'no', 'nb', 'nn',  # Norwegian variants
+        }
+        valid_codes.update(django_special)
+
+        cls._valid_codes_cache = valid_codes
+        return valid_codes
+
     @staticmethod
-    def detect_languages_from_folder(folder: str, use_folder_structure: bool = False) -> List[str]:
+    def detect_languages_from_folder(folder: str, use_folder_structure: bool = False,
+                                     respect_gitignore: bool = True) -> List[str]:
         """
         Scan all PO files in a folder and detect their languages.
 
         Args:
             folder: Path to folder containing PO files
             use_folder_structure: If True, detect languages from folder names instead of metadata
+            respect_gitignore: If True, respect .gitignore patterns when scanning
 
         Returns:
             List of unique language codes found in PO files
@@ -29,46 +82,120 @@ def detect_languages_from_folder(folder: str, use_folder_structure: bool = False
             ValueError: If no languages could be detected
         """
         if use_folder_structure:
-            return LanguageDetector._detect_from_folder_structure(folder)
+            return LanguageDetector._detect_from_folder_structure(folder, respect_gitignore)
 
-        return LanguageDetector._detect_from_metadata(folder)
+        return LanguageDetector._detect_from_metadata(folder, respect_gitignore)
 
     @staticmethod
-    def _detect_from_folder_structure(folder: str) -> List[str]:
-        """Detect languages from folder structure (e.g., locale/it/LC_MESSAGES/)."""
+    def _detect_from_folder_structure(folder: str, respect_gitignore: bool = True) -> List[str]:
+        """Detect languages from folder structure based on common framework patterns."""
         languages = set()
         po_files_found = 0
+        valid_codes = LanguageDetector._get_valid_language_codes()
+        gitignore_parser = create_gitignore_parser(folder, respect_gitignore)
 
-        # Scan all PO files
-        for root, _, files in os.walk(folder):
-            for file in files:
-                if not file.endswith('.po'):
-                    continue
+        for root, dirs, files in os.walk(folder):
+            dirs[:], files = gitignore_parser.filter_walk_results(root, dirs, files)
+            po_files_in_dir = [f for f in files if f.endswith('.po')]
 
-                po_files_found += 1
-                po_path = os.path.join(root, file)
+            if not po_files_in_dir:
+                continue
 
-                # Extract language code from path
-                # Common patterns: locale/it/LC_MESSAGES/, it/, locale/it/, etc.
-                relative_path = os.path.relpath(po_path, folder)
+            for file in po_files_in_dir:
+                po_files_found += 1
+                relative_path = os.path.relpath(os.path.join(root, file), folder)
                 path_parts = relative_path.split(os.sep)
 
-                # Look for language codes in path components
-                for part in path_parts:
-                    # Skip common directory names
-                    if part in ['locale', 'locales', 'LC_MESSAGES', 'po', 'i18n', 'translations']:
-                        continue
-                    if part.endswith('.po'):
-                        continue
+                detected_lang = LanguageDetector._detect_language_from_path(
+                    path_parts, file, valid_codes
+                )
+
+                if detected_lang:
+                    languages.add(detected_lang)
+                    logging.debug("Found language '%s' from path %s", detected_lang,
+                                  os.path.join(root, file))
+
+        LanguageDetector._validate_detection_results(po_files_found, languages)
+        detected = sorted(list(languages))
+        logging.info("Auto-detected languages from folder structure in %d PO files: %s",
+                     po_files_found, ', '.join(detected))
+        return detected
+
+    @staticmethod
+    def _detect_language_from_path(path_parts: List[str], filename: str,
+                                   valid_codes: set) -> Optional[str]:
+        """Detect language from a single file path using various patterns."""
+        # Pattern 1: LC_MESSAGES structure
+        detected_lang = LanguageDetector._detect_from_lc_messages(path_parts, valid_codes)
+        if detected_lang:
+            return detected_lang
+
+        # Pattern 2: WordPress filename pattern
+        detected_lang = LanguageDetector._detect_from_filename(filename, valid_codes)
+        if detected_lang:
+            return detected_lang
+
+        # Pattern 3: Directory structure
+        detected_lang = LanguageDetector._detect_from_directories(path_parts, valid_codes)
+        if detected_lang:
+            return detected_lang
+
+        # Pattern 4: Flat structure
+        return LanguageDetector._detect_from_flat_structure(filename, valid_codes)
+
+    @staticmethod
+    def _detect_from_lc_messages(path_parts: List[str], valid_codes: set) -> Optional[str]:
+        """Detect language from LC_MESSAGES pattern."""
+        if 'LC_MESSAGES' in path_parts:
+            lc_idx = path_parts.index('LC_MESSAGES')
+            if lc_idx > 0:
+                potential_lang = path_parts[lc_idx - 1]
+                if potential_lang in valid_codes or potential_lang.lower() in valid_codes:
+                    return potential_lang
+        return None
+
+    @staticmethod
+    def _detect_from_filename(filename: str, valid_codes: set) -> Optional[str]:
+        """Detect language from WordPress-style filename pattern."""
+        if '-' in filename:
+            parts = filename.replace('.po', '').split('-')
+            if len(parts) >= 2:
+                potential_lang = parts[-1]
+                if potential_lang in valid_codes or potential_lang.lower() in valid_codes:
+                    return potential_lang
+        return None
+
+    @staticmethod
+    def _detect_from_directories(path_parts: List[str], valid_codes: set) -> Optional[str]:
+        """Detect language from directory structure."""
+        locale_dirs = ['locale', 'locales', 'i18n', 'translations', 'lang', 'languages', 'po']
+
+        for i, part in enumerate(path_parts[:-1]):  # Exclude filename
+            if part in locale_dirs:
+                continue
+
+            if part in valid_codes or part.lower() in valid_codes:
+                prev_is_locale = i > 0 and path_parts[i - 1] in locale_dirs
+                is_po_parent = i == len(path_parts) - 2
 
-                    # Check if this looks like a language code
-                    if LanguageDetector._is_language_code(part):
-                        languages.add(part)
-                        logging.debug("Found language '%s' from path %s", part, po_path)
-                        break
+                if prev_is_locale or is_po_parent:
+                    return part
+        return None
 
+    @staticmethod
+    def _detect_from_flat_structure(filename: str, valid_codes: set) -> Optional[str]:
+        """Detect language from flat structure where filename is the language code."""
+        if filename.endswith('.po'):
+            lang_candidate = filename.replace('.po', '')
+            if lang_candidate in valid_codes or lang_candidate.lower() in valid_codes:
+                return lang_candidate
+        return None
+
+    @staticmethod
+    def _validate_detection_results(po_files_found: int, languages: set):
+        """Validate detection results and raise appropriate errors."""
         if not po_files_found:
-            raise ValueError(f"No .po files found in folder: {folder}")
+            raise ValueError("No .po files found in folder")
 
         if not languages:
             raise ValueError(
@@ -77,23 +204,20 @@ def _detect_from_folder_structure(folder: str) -> List[str]:
                 f"or use -l to specify languages."
             )
 
-        # Convert to sorted list for consistent ordering
-        detected = sorted(list(languages))
-
-        logging.info("Auto-detected languages from folder structure in %d PO files: %s",
-                     po_files_found, ', '.join(detected))
-
-        return detected
-
     @staticmethod
-    def _detect_from_metadata(folder: str) -> List[str]:
+    def _detect_from_metadata(folder: str, respect_gitignore: bool = True) -> List[str]:
         """Detect languages from PO file metadata."""
         languages = set()
         po_files_found = 0
         files_with_language = 0
 
+        # Create gitignore parser for filtering
+        gitignore_parser = create_gitignore_parser(folder, respect_gitignore)
+
         # Scan all PO files
-        for root, _, files in os.walk(folder):
+        for root, dirs, files in os.walk(folder):
+            # Filter directories and files using gitignore parser
+            dirs[:], files = gitignore_parser.filter_walk_results(root, dirs, files)
             for file in files:
                 if not file.endswith('.po'):
                     continue
@@ -137,38 +261,36 @@ def _detect_from_metadata(folder: str) -> List[str]:
 
     @staticmethod
     def _is_language_code(code: str) -> bool:
-        """Check if a string looks like a language code."""
-        # Basic validation
-        if not code or len(code) < 2 or len(code) > 10 or not code[0].isalpha():
+        """Check if a string is a valid language code."""
+        if not code or len(code) < 2:
             return False
 
-        # Common language code patterns:
-        # - 2-letter codes: en, fr, de, etc.
-        # - 2+2 codes: en_US, fr_FR, etc.
-        # - Special codes: zh_Hans, sr_Latn, be@tarask, etc.
+        # Get valid codes
+        valid_codes = LanguageDetector._get_valid_language_codes()
 
-        # Special cases used in Django
-        special_codes = {
-            'zh-hans', 'zh-hant', 'sr-latn', 'sr-cyrl', 'az-latn', 'az-cyrl',
-            'uz-latn', 'uz-cyrl', 'kk-latn', 'kk-cyrl', 'ky-latn', 'ky-cyrl'
-        }
+        # Check if code is valid (case-insensitive)
+        if code in valid_codes or code.lower() in valid_codes:
+            return True
+
+        # Check with normalization
+        normalized = code.replace('-', '_')
+        if normalized in valid_codes or normalized.lower() in valid_codes:
+            return True
 
-        # Check all valid patterns
-        pattern_match = re.match(r'^[a-z]{2,3}(_[A-Z][a-z]+|_[A-Z]{2}|@[a-z]+)?$', code) is not None
-        in_special = code.lower() in special_codes
-        basic_match = re.match(r'^[a-z]{2,3}$', code) is not None
-        return pattern_match or in_special or basic_match
+        return False
 
     @staticmethod
     def validate_or_detect_languages(folder: str, lang_arg: str = None,
-                                     use_folder_structure: bool = False) -> List[str]:
+                                     use_folder_structure: bool = False,
+                                     respect_gitignore: bool = True) -> List[str]:
         """
         Get languages from command line or auto-detect from PO files.
 
         Args:
             folder: Path to folder containing PO files
             lang_arg: Language argument from command line (optional)
             use_folder_structure: If True, detect languages from folder names instead of metadata
+            respect_gitignore: If True, respect .gitignore patterns when scanning
 
         Returns:
             List of language codes to process
@@ -187,7 +309,7 @@ def validate_or_detect_languages(folder: str, lang_arg: str = None,
         logging.info("No languages specified with -l, auto-detecting from %s...", detection_method)
 
         try:
-            return LanguageDetector.detect_languages_from_folder(folder, use_folder_structure)
+            return LanguageDetector.detect_languages_from_folder(folder, use_folder_structure, respect_gitignore)
         except ValueError as e:
             # Re-raise with more helpful message
             if use_folder_structure:

python_gpt_po/services/providers/__init__.py

@@ -1,7 +1,6 @@
 """
 Provider implementations for model management.
 """
-from . import provider_init  # noqa: F401 - Auto-registers providers
 from .base import ModelProviderInterface
 from .registry import ProviderRegistry