Implement visual selector override of price data Re #3505

changedetectionio/blueprint/ui/templates/edit.html

@@ -29,7 +29,7 @@
     const default_system_fetch_backend="{{ settings_application['fetch_backend'] }}";
 </script>
 <script src="{{url_for('static_content', group='js', filename='plugins.js')}}" defer></script>
-<script src="{{url_for('static_content', group='js', filename='watch-settings.js')}}" defer></script>
+<script src="{{url_for('static_content', group='js', filename=watch['processor']+".js")}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='notifications.js')}}" defer></script>
 <script src="{{url_for('static_content', group='js', filename='visual-selector.js')}}" defer></script>
 {% if playwright_enabled %}
@@ -50,8 +50,10 @@
             {% endif %}
             <li class="tab"><a id="browsersteps-tab" href="#browser-steps">Browser Steps</a></li>
         <!-- should goto extra forms? -->
-            {% if watch['processor'] == 'text_json_diff' %}
+            {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'restock_diff' %}
             <li class="tab"><a id="visualselector-tab" href="#visualselector">Visual Filter Selector</a></li>
+            {% endif %}
+            {% if watch['processor'] == 'text_json_diff' %}
             <li class="tab" id="filters-and-triggers-tab"><a href="#filters-and-triggers">Filters &amp; Triggers</a></li>
             <li class="tab" id="conditions-tab"><a href="#conditions">Conditions</a></li>
             {% endif %}
@@ -377,7 +379,7 @@ <h3>Text filtering</h3>
             {{ extra_form_content|safe }}
             </div>
         {% endif %}
-            {% if watch['processor'] == 'text_json_diff' %}
+            {% if watch['processor'] == 'text_json_diff' or watch['processor'] == 'restock_diff' %}
             <div class="tab-pane-inner visual-selector-ui" id="visualselector">
                 <img class="beta-logo" src="{{url_for('static_content', group='images', filename='beta-logo.png')}}" alt="New beta functionality">
 

changedetectionio/model/__init__.py

@@ -55,6 +55,7 @@ def __init__(self, *arg, **kw):
             'previous_md5_before_filters': False,  # Used for skipping changedetection entirely
             'processor': 'text_json_diff',  # could be restock_diff or others from .processors
             'price_change_threshold_percent': None,
+            'price_change_custom_include_filters': None, # Like 'include_filter' but for price changes only
             'proxy': None,  # Preferred proxy connection
             'remote_server_reply': None,  # From 'server' reply header
             'sort_text_alphabetically': False,

changedetectionio/processors/restock_diff/__init__.py

@@ -6,8 +6,69 @@
 
 class Restock(dict):
 
-    def parse_currency(self, raw_value: str) -> Union[float, None]:
-        # Clean and standardize the value (ie 1,400.00 should be 1400.00), even better would be store the whole thing as an integer.
+    def _normalize_currency_code(self, currency: str, normalize_dollar=False) -> str:
+        """
+        Normalize currency symbol or code to ISO 4217 code for consistency.
+        Uses iso4217parse for accurate conversion.
+
+        Returns empty string for ambiguous symbols like '$' where we can't determine
+        the specific currency (USD, CAD, AUD, etc.).
+        """
+        if not currency:
+            return currency
+
+        # If already a 3-letter code, likely already normalized
+        if len(currency) == 3 and currency.isupper():
+            return currency
+
+        # Handle ambiguous dollar sign - can't determine which dollar currency
+        if normalize_dollar and currency == '$':
+            return ''
+
+        try:
+            import iso4217parse
+
+            # Parse the currency - returns list of possible matches
+            # This handles: € -> EUR, Kč -> CZK, £ -> GBP, ¥ -> JPY, etc.
+            currencies = iso4217parse.parse(currency)
+
+            if currencies:
+                # Return first match (iso4217parse handles the mapping)
+                return currencies[0].alpha3
+        except Exception:
+            pass
+
+        # Fallback: return as-is if can't normalize
+        return currency
+
+    def parse_currency(self, raw_value: str, normalize_dollar=False) -> Union[dict, None]:
+        """
+        Parse price and currency from text, handling messy formats with extra text.
+        Returns dict with 'price' and 'currency' keys (ISO 4217 code), or None if parsing fails.
+
+        normalize_dollar convert $ to '' on sites that we cant tell what currency the site is in
+        """
+        try:
+            from price_parser import Price
+            # price-parser handles:
+            # - Extra text before/after ("Beginning at", "tax incl.")
+            # - Various number formats (1 099,00 or 1,099.00)
+            # - Currency symbols and codes
+            price_obj = Price.fromstring(raw_value)
+
+            if price_obj.amount is not None:
+                result = {'price': float(price_obj.amount)}
+                if price_obj.currency:
+                    # Normalize currency symbol to ISO 4217 code for consistency with metadata
+                    normalized_currency = self._normalize_currency_code(currency=price_obj.currency, normalize_dollar=normalize_dollar)
+                    result['currency'] = normalized_currency
+                return result
+
+        except Exception as e:
+            from loguru import logger
+            logger.trace(f"price-parser failed on '{raw_value}': {e}, falling back to manual parsing")
+
+        # Fallback to existing manual parsing logic
         standardized_value = raw_value
 
         if ',' in standardized_value and '.' in standardized_value:
@@ -24,7 +85,7 @@ def parse_currency(self, raw_value: str) -> Union[float, None]:
 
         if standardized_value:
             # Convert to float
-            return float(parse_decimal(standardized_value, locale='en'))
+            return {'price': float(parse_decimal(standardized_value, locale='en'))}
 
         return None
 
@@ -51,7 +112,15 @@ def __setitem__(self, key, value):
         # Custom logic to handle setting price and original_price
         if key == 'price' or key == 'original_price':
             if isinstance(value, str):
-                value = self.parse_currency(raw_value=value)
+                parsed = self.parse_currency(raw_value=value)
+                if parsed:
+                    # Set the price value
+                    value = parsed.get('price')
+                    # Also set currency if found and not already set
+                    if parsed.get('currency') and not self.get('currency'):
+                        super().__setitem__('currency', parsed.get('currency'))
+                else:
+                    value = None
 
         super().__setitem__(key, value)
 

changedetectionio/processors/restock_diff/forms.py

@@ -1,13 +1,13 @@
 from wtforms import (
     BooleanField,
     validators,
-    FloatField
+    FloatField, StringField
 )
 from wtforms.fields.choices import RadioField
 from wtforms.fields.form import FormField
 from wtforms.form import Form
 
-from changedetectionio.forms import processor_text_json_diff_form
+from changedetectionio.forms import processor_text_json_diff_form, ValidateCSSJSONXPATHInput, StringListField
 
 
 class RestockSettingsForm(Form):
@@ -27,6 +27,8 @@ class RestockSettingsForm(Form):
         validators.NumberRange(min=0, max=100, message="Should be between 0 and 100"),
     ], render_kw={"placeholder": "0%", "size": "5"})
 
+    price_change_custom_include_filters = StringField('Override automatic price detection with this selector', [ValidateCSSJSONXPATHInput()], default='', render_kw={"style": "width: 100%;"})
+
     follow_price_changes = BooleanField('Follow price changes', default=True)
 
 class processor_settings_form(processor_text_json_diff_form):
@@ -74,7 +76,11 @@ def extra_form_content(self):
                     {{ render_field(form.restock_settings.price_change_threshold_percent) }}
                     <span class="pure-form-message-inline">Price must change more than this % to trigger a change since the first check.</span><br>
                     <span class="pure-form-message-inline">For example, If the product is $1,000 USD originally, <strong>2%</strong> would mean it has to change more than $20 since the first check.</span><br>
-                </fieldset>                
+                </fieldset>
+                <fieldset class="pure-group price-change-minmax">
+                        {{ render_field(form.restock_settings.price_change_custom_include_filters) }}
+                        <span class="pure-form-message-inline">Override the automatic price metadata reader with this custom select from the <a href="#visualselector">Visual Selector</a>, in the case that the automatic detection was incorrect.</span><br>
+                </fieldset>                                  
             </div>
         </fieldset>
         """

changedetectionio/processors/restock_diff/processor.py

@@ -6,6 +6,8 @@
 import urllib3
 import time
 
+from ..text_json_diff.processor import FilterNotFoundInResponse
+
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 name = 'Re-stock & Price detection for pages with a SINGLE product'
 description = 'Detects if the product goes back to in-stock'
@@ -125,6 +127,85 @@ def get_itemprop_availability(html_content) -> Restock:
     return value
 
 
+def get_price_data_availability_from_filters(html_content, price_change_custom_include_filters) -> Restock:
+    """
+    Extract price using custom CSS/XPath selectors.
+    Reuses apply_include_filters logic from text_json_diff processor.
+
+    Args:
+        html_content: The HTML content to parse
+        price_change_custom_include_filters: List of CSS/XPath selectors to extract price
+
+    Returns:
+        Restock dict with 'price' key if found
+    """
+    from changedetectionio import html_tools
+    from changedetectionio.processors.magic import guess_stream_type
+
+    value = Restock()
+
+    if not price_change_custom_include_filters:
+        return value
+
+    # Get content type
+    stream_content_type = guess_stream_type(http_content_header='text/html', content=html_content)
+
+    # Apply filters to extract price element
+    filtered_content = ""
+
+    for filter_rule in price_change_custom_include_filters:
+        # XPath filters
+        if filter_rule[0] == '/' or filter_rule.startswith('xpath:'):
+            filtered_content += html_tools.xpath_filter(
+                xpath_filter=filter_rule.replace('xpath:', ''),
+                html_content=html_content,
+                append_pretty_line_formatting=False,
+                is_rss=stream_content_type.is_rss
+            )
+
+        # XPath1 filters (first match only)
+        elif filter_rule.startswith('xpath1:'):
+            filtered_content += html_tools.xpath1_filter(
+                xpath_filter=filter_rule.replace('xpath1:', ''),
+                html_content=html_content,
+                append_pretty_line_formatting=False,
+                is_rss=stream_content_type.is_rss
+            )
+
+        # CSS selectors, default fallback
+        else:
+            filtered_content += html_tools.include_filters(
+                include_filters=filter_rule,
+                html_content=html_content,
+                append_pretty_line_formatting=False
+            )
+
+    if filtered_content.strip():
+        # Convert HTML to text
+        import re
+        price_text = re.sub(
+            r'[\r\n\t]+', ' ',
+            html_tools.html_to_text(
+                html_content=filtered_content,
+                render_anchor_tag_content=False,
+                is_rss=False
+            ).strip()
+        )
+
+        # Parse the price from text
+        try:
+            parsed_result = value.parse_currency(price_text, normalize_dollar=True)
+            if parsed_result:
+                value['price'] = parsed_result.get('price')
+                if parsed_result.get('currency'):
+                    value['currency'] = parsed_result.get('currency')
+                logger.debug(f"Extracted price from custom selector: {parsed_result.get('price')} {parsed_result.get('currency', '')} (from text: '{price_text}')")
+        except Exception as e:
+            logger.warning(f"Failed to parse price from '{price_text}': {e}")
+
+    return value
+
+
 def is_between(number, lower=None, upper=None):
     """
     Check if a number is between two values.
@@ -185,18 +266,32 @@ def run_changedetection(self, watch):
                 logger.info(f"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override")
                 break
 
-
+        # if not has custom selector..
         itemprop_availability = {}
-        try:
-            itemprop_availability = get_itemprop_availability(self.fetcher.content)
-        except MoreThanOnePriceFound as e:
-            # Add the real data
-            raise ProcessorException(message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
-                                     url=watch.get('url'),
-                                     status_code=self.fetcher.get_last_status_code(),
-                                     screenshot=self.fetcher.screenshot,
-                                     xpath_data=self.fetcher.xpath_data
-                                     )
+        if restock_settings.get('price_change_custom_include_filters'):
+            itemprop_availability = get_price_data_availability_from_filters(html_content=self.fetcher.content,
+                                                                             price_change_custom_include_filters=restock_settings.get(
+                                                                                 'price_change_custom_include_filters')
+                                                                             )
+            if not itemprop_availability or not itemprop_availability.get('price'):
+                raise FilterNotFoundInResponse(
+                    msg=restock_settings.get('price_change_custom_include_filters'),
+                    screenshot=self.fetcher.screenshot,
+                    xpath_data=self.fetcher.xpath_data
+                )
+
+        else:
+            try:
+                itemprop_availability = get_itemprop_availability(self.fetcher.content)
+            except MoreThanOnePriceFound as e:
+                # Add the real data
+                raise ProcessorException(
+                    message="Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.",
+                    url=watch.get('url'),
+                    status_code=self.fetcher.get_last_status_code(),
+                    screenshot=self.fetcher.screenshot,
+                    xpath_data=self.fetcher.xpath_data
+                    )
 
         # Something valid in get_itemprop_availability() by scraping metadata ?
         if itemprop_availability.get('price') or itemprop_availability.get('availability'):

changedetectionio/static/js/restock_diff.js

@@ -0,0 +1,42 @@
+$(document).ready(function () {
+    // Initialize Visual Selector plugin
+    let visualSelectorAPI = null;
+    if ($('#selector-wrapper').length > 0) {
+        visualSelectorAPI = $('#selector-wrapper').visualSelector({
+            screenshotUrl: screenshot_url,
+            visualSelectorDataUrl: watch_visual_selector_data_url,
+            singleSelectorOnly: true,
+            $includeFiltersElem: $('#restock_settings-price_change_custom_include_filters')
+        });
+    }
+
+    // Function to check and bootstrap visual selector based on hash
+    function checkAndBootstrapVisualSelector() {
+        if (visualSelectorAPI) {
+            if (window.location.hash && window.location.hash.includes('visualselector')) {
+                $('img#selector-background').off('load');
+                visualSelectorAPI.bootstrap();
+            } else {
+                // Shutdown when navigating away from visualselector
+                visualSelectorAPI.shutdown();
+            }
+        }
+    }
+
+    // Bootstrap the visual selector when the tab is clicked
+    $('#visualselector-tab').click(function () {
+        if (visualSelectorAPI) {
+            $('img#selector-background').off('load');
+            visualSelectorAPI.bootstrap();
+        }
+    });
+
+    // Check on page load if hash contains 'visualselector'
+    checkAndBootstrapVisualSelector();
+
+    // Listen for hash changes (when anchor changes)
+    $(window).on('hashchange', function() {
+        checkAndBootstrapVisualSelector();
+    });
+});
+