Merge pull request #90 from ServiceNow/fix-issue-86

Fix issue with date filters for reports

Author: aldro61

src/browsergym/workarena/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.4.3"
+__version__ = "0.4.4"
 
 import inspect
 from logging import warning

src/browsergym/workarena/api/system_properties.py

@@ -0,0 +1,66 @@
+from .utils import table_api_call
+
+
+def set_sys_property(instance, property_name: str, value: str):
+    """
+    Set a sys_property in the instance.
+
+    Parameters:
+    -----------
+    instance: SNowInstance
+        The instance to set the property in
+    property_name: str
+        The name of the property to set
+    value: str
+        The value to set for the property
+
+    """
+
+    property = table_api_call(
+        instance=instance,
+        table="sys_properties",
+        params={"sysparm_query": f"name={property_name}", "sysparm_fields": "sys_id"},
+    )["result"]
+
+    if not property:
+        property_sysid = ""
+        method = "POST"
+    else:
+        property_sysid = "/" + property[0]["sys_id"]
+        method = "PUT"
+
+    property = table_api_call(
+        instance=instance,
+        table=f"sys_properties{property_sysid}",
+        method=method,
+        json={"name": property_name, "value": value},
+    )
+
+    # Verify that the property was updated
+    assert property["result"]["value"] == value, f"Error setting {property_name}."
+
+
+def get_sys_property(instance, property_name: str) -> str:
+    """
+    Get a sys_property from the instance.
+
+    Parameters:
+    -----------
+    instance: SNowInstance
+        The instance to get the property from
+    property_name: str
+        The name of the property to get
+
+    Returns:
+    --------
+    str
+        The value of the property
+
+    """
+    property_value = table_api_call(
+        instance=instance,
+        table="sys_properties",
+        params={"sysparm_query": f"name={property_name}", "sysparm_fields": "value"},
+    )["result"][0]["value"]
+
+    return property_value

src/browsergym/workarena/config.py

@@ -1,4 +1,6 @@
 from importlib import resources
+from json import load as json_load
+from os.path import exists
 
 from ..workarena import data_files
 from ..workarena.tasks import utils
@@ -224,4 +226,4 @@
 
 # Report date filter patch flag
 REPORT_PATCH_FLAG = "WORKARENA_DATE_FILTER_PATCH"
-REPORT_DATE_FILTER = "2025-07-15"
+REPORT_FILTER_PROPERTY = "workarena.report.filter.config"

src/browsergym/workarena/data_files/task_configs/report_retrieval_minmax_task.json

@@ -10,6 +10,7 @@
 from requests import HTTPError
 from time import sleep
 
+from .api.system_properties import get_sys_property, set_sys_property
 from .api.ui_themes import get_workarena_theme_variants
 from .api.user import create_user
 from .api.utils import table_api_call, table_column_info
@@ -38,7 +39,7 @@
     EXPECTED_USER_FORM_FIELDS_PATH,
     # Patch flag for reports
     REPORT_PATCH_FLAG,
-    REPORT_DATE_FILTER,
+    REPORT_FILTER_PROPERTY,
     # Supported ServiceNow releases
     SNOW_SUPPORTED_RELEASES,
     # For workflows setup
@@ -51,51 +52,22 @@
 from .utils import url_login
 
 
-def _set_sys_property(property_name: str, value: str):
+def _is_dev_portal_instance() -> bool:
     """
-    Set a sys_property in the instance.
+    Check if the instance is a ServiceNow Developer Portal instance.
 
-    """
-    instance = SNowInstance()
-
-    property = table_api_call(
-        instance=instance,
-        table="sys_properties",
-        params={"sysparm_query": f"name={property_name}", "sysparm_fields": "sys_id"},
-    )["result"]
-
-    if not property:
-        property_sysid = ""
-        method = "POST"
-    else:
-        property_sysid = "/" + property[0]["sys_id"]
-        method = "PUT"
-
-    property = table_api_call(
-        instance=instance,
-        table=f"sys_properties{property_sysid}",
-        method=method,
-        json={"name": property_name, "value": value},
-    )
-
-    # Verify that the property was updated
-    assert property["result"]["value"] == value, f"Error setting {property_name}."
-
-
-def _get_sys_property(property_name: str) -> str:
-    """
-    Get a sys_property from the instance.
+    Returns:
+    --------
+    bool: True if the instance is a developer portal instance, False otherwise.
 
     """
     instance = SNowInstance()
-
-    property_value = table_api_call(
-        instance=instance,
-        table="sys_properties",
-        params={"sysparm_query": f"name={property_name}", "sysparm_fields": "value"},
-    )["result"][0]["value"]
-
-    return property_value
+    # Check if the instance url has the for devXXXXXX.service-now.com format (where X is a digit)
+    if re.match(r"^https?://dev\d{6}\.service-now\.com", instance.snow_url):
+        logging.info("Detected a developer portal instance...")
+        return True
+    logging.info("Detected an internal instance...")
+    return False
 
 
 def _install_update_set(path: str, name: str):
@@ -797,7 +769,9 @@ def enable_url_login():
     Configure the instance to allow login via URL.
 
     """
-    _set_sys_property(property_name="glide.security.restrict.get.login", value="false")
+    set_sys_property(
+        instance=SNowInstance(), property_name="glide.security.restrict.get.login", value="false"
+    )
     logging.info("URL login enabled.")
 
 
@@ -808,7 +782,21 @@ def disable_password_policies():
     Notes: this is required to allow the creation of users with weak passwords.
 
     """
-    _set_sys_property(property_name="glide.security.password.policy.enabled", value="false")
+    set_sys_property(
+        instance=SNowInstance(),
+        property_name="glide.security.password.policy.enabled",
+        value="false",
+    )
+    set_sys_property(
+        instance=SNowInstance(), property_name="glide.apply.password_policy.on_login", value="false"
+    )
+    # The following is not supported on developer portal instances
+    if not _is_dev_portal_instance():
+        set_sys_property(
+            instance=SNowInstance(),
+            property_name="glide.authenticate.api.user.reset_password.mandatory",
+            value="false",
+        )
     logging.info("Password policies disabled.")
 
 
@@ -817,8 +805,14 @@ def disable_guided_tours():
     Hide guided tour popups
 
     """
-    _set_sys_property(property_name="com.snc.guided_tours.sp.enable", value="false")
-    _set_sys_property(property_name="com.snc.guided_tours.standard_ui.enable", value="false")
+    set_sys_property(
+        instance=SNowInstance(), property_name="com.snc.guided_tours.sp.enable", value="false"
+    )
+    set_sys_property(
+        instance=SNowInstance(),
+        property_name="com.snc.guided_tours.standard_ui.enable",
+        value="false",
+    )
     logging.info("Guided tours disabled.")
 
 
@@ -836,7 +830,9 @@ def disable_analytics_popups():
     Disable analytics popups (needs to be done through UI since Vancouver release)
 
     """
-    _set_sys_property(property_name="glide.analytics.enabled", value="false")
+    set_sys_property(
+        instance=SNowInstance(), property_name="glide.analytics.enabled", value="false"
+    )
     logging.info("Analytics popups disabled.")
 
 
@@ -850,7 +846,8 @@ def setup_ui_themes():
     check_ui_themes_installed()
 
     logging.info("Setting default UI theme")
-    _set_sys_property(
+    set_sys_property(
+        instance=SNowInstance(),
         property_name="glide.ui.polaris.theme.custom",
         value=get_workarena_theme_variants(SNowInstance())[0]["theme.sys_id"],
     )
@@ -894,7 +891,9 @@ def check_ui_themes_installed():
 
 def set_home_page():
     logging.info("Setting default home page")
-    _set_sys_property(property_name="glide.login.home", value="/now/nav/ui/home")
+    set_sys_property(
+        instance=SNowInstance(), property_name="glide.login.home", value="/now/nav/ui/home"
+    )
 
 
 def wipe_system_admin_preferences():
@@ -918,9 +917,9 @@ def wipe_system_admin_preferences():
         )
 
 
-def is_report_filter_using_time(filter):
+def is_report_filter_using_relative_time(filter):
     """
-    Heuristic to check if a report is filtering based on time
+    Heuristic to check if a report is filtering based on relative time
 
     This aims to detect the use of functions like "gs.endOfToday()". To avoid hardcoding all of them,
     we simply check for the use of keywords. Our filter is definitely too wide, but that's ok.
@@ -938,6 +937,32 @@ def patch_report_filters():
     logging.info("Patching reports with date filter...")
 
     instance = SNowInstance()
+    filter_config = instance.report_filter_config
+
+    # If the report date filter is already set, we use the existing values (would be the case on reinstall)
+    if not filter_config:
+        # Set the report date filter to current date as YYYY-MM-DD and time filter to current time as HH:MM:SS
+        now = datetime.now()
+        report_date_filter = now.strftime("%Y-%m-%d")
+        report_time_filter = now.strftime("%H:%M:%S")
+        # ... save the filter config
+        logging.info(
+            f"Setting report date filter to {report_date_filter} and time filter to {report_time_filter} via {REPORT_FILTER_PROPERTY}"
+        )
+        set_sys_property(
+            instance=instance,
+            property_name=REPORT_FILTER_PROPERTY,
+            value=json.dumps(
+                {"report_date_filter": report_date_filter, "report_time_filter": report_time_filter}
+            ),
+        )
+    else:
+        # Use the existing configuration
+        logging.info(
+            f"Using existing report date filter {filter_config['report_date_filter']} and time filter {filter_config['report_time_filter']}"
+        )
+        report_date_filter = filter_config["report_date_filter"]
+        report_time_filter = filter_config["report_time_filter"]
 
     # Get all reports that are not already patched
     reports = table_api_call(
@@ -959,27 +984,31 @@ def patch_report_filters():
                 logging.info(f"Discarding report {report['title']} {report['sys_id']}...")
                 raise NotImplementedError()  # Mark for deletion
 
-            if not is_report_filter_using_time(report["filter"]):
+            if not is_report_filter_using_relative_time(report["filter"]):
                 # That's a report we want to keep (use date cutoff filter)
-                filter_date = REPORT_DATE_FILTER
+                filter_date = report_date_filter
+                filter_time = report_time_filter
                 logging.info(
                     f"Keeping report {report['title']} {report['sys_id']} (columns: {sys_created_on_cols})..."
                 )
             else:
-                # XXX: We do not support reports with filters that rely on time (e.g., last 10 days) because
+                # XXX: We do not support reports with filters that rely on relative time (e.g., last 10 days) because
                 #      there are not stable. In this case, we don't delete them but add a filter to make
                 #      them empty. They will be shown as "No data available".
                 logging.info(
                     f"Disabling report {report['title']} {report['sys_id']} because it uses time filters..."
                 )
                 filter_date = "1900-01-01"
+                filter_time = "00:00:00"
 
+            # Format the filter
             filter = "".join(
                 [
-                    f"^{col}<javascript:gs.dateGenerate('{filter_date}','00:00:00')"
+                    f"^{col}<javascript:gs.dateGenerate('{filter_date}','{filter_time}')"
                     for col in sys_created_on_cols
                 ]
             ) + ("^" if len(report["filter"]) > 0 and not report["filter"].startswith("^") else "")
+            # Patch the report with the new filter
             table_api_call(
                 instance=instance,
                 table=f"sys_report/{report['sys_id']}",
@@ -1055,7 +1084,11 @@ def setup():
 
     # Save installation date
     logging.info("Saving installation date")
-    _set_sys_property(property_name="workarena.installation.date", value=datetime.now().isoformat())
+    set_sys_property(
+        instance=SNowInstance(),
+        property_name="workarena.installation.date",
+        value=datetime.now().isoformat(),
+    )
 
     logging.info("WorkArena setup complete.")
 
@@ -1068,7 +1101,9 @@ def main():
     logging.basicConfig(level=logging.INFO)
 
     try:
-        past_install_date = _get_sys_property("workarena.installation.date")
+        past_install_date = get_sys_property(
+            instance=SNowInstance(), property_name="workarena.installation.date"
+        )
         logging.info(f"Detected previous installation on {past_install_date}. Reinstalling...")
     except:
         past_install_date = "never"

src/browsergym/workarena/data_files/task_configs/report_retrieval_value_task.json

@@ -1,11 +1,12 @@
+import json
 import os
 import requests
 import re
 
 from playwright.sync_api import sync_playwright
 from typing import Optional
 
-from .config import SNOW_BROWSER_TIMEOUT
+from .config import SNOW_BROWSER_TIMEOUT, REPORT_FILTER_PROPERTY
 
 
 class SNowInstance:
@@ -124,3 +125,25 @@ def release_version(self) -> str:
             browser.close()
 
         return release_info
+
+    @property
+    def report_filter_config(self) -> dict:
+        """
+        Get the report filter configuration from the ServiceNow instance.
+
+        Returns:
+        --------
+        dict
+            The report filter configuration, or an empty dictionary if not found.
+
+        """
+        from .api.system_properties import (
+            get_sys_property,
+        )  # Import here to avoid circular import issues
+
+        try:
+            config = get_sys_property(self, REPORT_FILTER_PROPERTY)
+            config = json.loads(config)
+            return config
+        except Exception:
+            return None