Added features to support new notebooklet development. (#456)

* Added new queries for Windows Events and Syslog
Added ability for cross provider query sharing
Added timeout feature to screenshot
Fixed issues with Alert Selector

* fixed data_providers merge issue

* Fixed issue with KQL Driver passing UA

* [fix] Doc fixes - TIProviders.rst, TimeSeriesAnomalies.rst

[update] DataQueries.rst new list.

* [update] Update version to v2.0.0.rc3

Co-authored-by: Pete Bryan <pebryan@microsoft.com>
Co-authored-by: Ian Hellen <ianhelle@microsoft.com>

Author: 3 people

docs/source/data_acquisition/DataQueries.rst

@@ -325,7 +325,7 @@ Pivot functions also call the `lookup_iocs` method of this class.
 
 Brief help is shown below. You can read more details about the
 attributes and functions in the
-[TILookup documentation](https://msticpy.readthedocs.io/api/msticpy.context.tilookup.html)
+:py:mod:`TILookup documentation <msticpy.context.tilookup>`
 
 To use TILookup, you need to create an instance of the class.
 Avoid creating lots of instances of this class:
@@ -528,7 +528,7 @@ Lookup an IoC from a single provider
 And show the output
 
 .. code-block:: ipython3
-   :emphasize-lines:1
+   :emphasize-lines: 1
 
     result, details = ti_lookup.lookup_ioc(observable="38.75.137.9", providers=["OTX"])
 

docs/source/data_acquisition/TIProviders.rst

@@ -33,8 +33,7 @@ TechCommunity blog posts.
   Time Series analysis & its applications in Security
   <https://techcommunity.microsoft.com/t5/azure-sentinel/looking-for-unknown-anomalies-what-is-normal-time-series/ba-p/555052>`__
 
-- `Time Series visualization of Palo Alto logs to detect data exfiltration
-   <https://techcommunity.microsoft.com/t5/azure-sentinel/time-series-visualization-of-palo-alto-logs-to-detect-data/ba-p/666344>`__
+- `Time Series visualization of Palo Alto logs to detect data exfiltration <https://techcommunity.microsoft.com/t5/azure-sentinel/time-series-visualization-of-palo-alto-logs-to-detect-data/ba-p/666344>`__
 
 Some useful background reading on Forecasting and prediction using time
 series here `Forecasting: Principals and Practice <https://otexts.com/fpp2/>`__

docs/source/visualization/TimeSeriesAnomalies.rst

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "2.0.0-pre2"
+VERSION = "2.0.0-pre3"

msticpy/_version.py

@@ -98,10 +98,10 @@ def screenshot(url: str, api_key: str = None) -> httpx.Response:
     )
     # Wait until the screenshot is ready and keep user updated with progress
     print("Getting screenshot")
-    progress = IntProgress(min=0, max=40)
+    progress = IntProgress(min=0, max=100)
     display.display(progress)
     ready = False
-    while not ready:
+    while not ready and progress.value < 100:
         progress.value += 1
         status_data = httpx.get(
             status_string, timeout=config.get_http_timeout(), headers=mp_ua_header()
@@ -111,9 +111,9 @@ def screenshot(url: str, api_key: str = None) -> httpx.Response:
             ready = True
         else:
             time.sleep(0.05)
-    progress.value = 40
+    progress.value = 100
 
-    # Once ready get the screenshot
+    # Once ready or timed out get the screenshot
     image_data = httpx.get(image_string, timeout=config.get_http_timeout())
 
     if image_data.status_code != 200:

msticpy/context/domain_utils.py

@@ -785,7 +785,7 @@ def _combine_results(
 
         if not result_list:
             print("No IoC matches")
-        return pd.concat(result_list, sort=False)
+        return pd.concat(result_list, sort=False) if result_list else None
 
 
 def _make_sync(future):

msticpy/context/tilookup.py

@@ -228,7 +228,9 @@ def lookup_iocs(
             )
             all_results.append(combined_results_df)
 
-        return pd.concat(all_results, ignore_index=True, sort=False, axis=0)
+        if all_results:
+            return pd.concat(all_results, ignore_index=True, sort=False, axis=0)
+        return pd.DataFrame()
 
     @staticmethod
     def _add_failure_status(src_ioc_frame: pd.DataFrame, lookup_status: LookupStatus):

msticpy/context/tiproviders/kql_base.py

@@ -29,6 +29,7 @@
 __author__ = "Ian Hellen"
 
 _DB_QUERY_FLAGS = ("print", "debug_query", "print_query")
+_COMPATIBLE_DRIVER_MAPPINGS = {"mssentinel": ["m365d"], "mde": ["m365d"]}
 
 
 @export
@@ -482,24 +483,30 @@ def _get_query_options(
             }
         return query_options
 
-    def _get_query_folder_for_env(self, root_path: str, environment: str) -> Path:
+    def _get_query_folder_for_env(self, root_path: str, environment: str) -> List[Path]:
         """Return query folder for current environment."""
-        environment = "MDE" if environment == "M365D" else environment
-        return self._resolve_package_path(root_path).joinpath(environment.casefold())
+        data_envs = [environment]
+        if environment.casefold() in _COMPATIBLE_DRIVER_MAPPINGS:
+            data_envs += _COMPATIBLE_DRIVER_MAPPINGS[environment.casefold()]
+        return [
+            self._resolve_package_path(root_path).joinpath(data_env.casefold())
+            for data_env in data_envs
+        ]
 
     def _read_queries_from_paths(self, query_paths) -> Dict[str, QueryStore]:
         """Fetch queries from YAML files in specified paths."""
         settings: Dict[str, Any] = config.settings.get(  # type: ignore
             "QueryDefinitions"
         )  # type: ignore
-        all_query_paths = []
+        all_query_paths: List[Union[Path, str]] = []
         for def_qry_path in settings.get("Default"):  # type: ignore
             # only read queries from environment folder
-            builtin_qry_path = self._get_query_folder_for_env(
+            builtin_qry_paths = self._get_query_folder_for_env(
                 def_qry_path, self.environment
             )
-            if builtin_qry_path and builtin_qry_path.is_dir():
-                all_query_paths.append(str(builtin_qry_path))
+            all_query_paths.extend(
+                str(qry_path) for qry_path in builtin_qry_paths if qry_path.is_dir()
+            )
 
         if settings.get("Custom") is not None:
             for custom_path in settings.get("Custom"):  # type: ignore
@@ -511,7 +518,6 @@ def _read_queries_from_paths(self, query_paths) -> Dict[str, QueryStore]:
                 param_qry_path = self._resolve_path(param_path)
                 if param_qry_path:
                     all_query_paths.append(param_qry_path)
-
         if all_query_paths:
             return QueryStore.import_files(
                 source_path=all_query_paths,

msticpy/data/core/data_providers.py

@@ -100,7 +100,7 @@ def __init__(self, connection_str: str = None, **kwargs):
         if not self._loaded:
             self._load_kql_magic()
 
-        self._set_kql_option("request_user_agent-tag", MSTICPY_USER_AGENT)
+        self._set_kql_option("request_user_agent_tag", MSTICPY_USER_AGENT)
         self._schema: Dict[str, Any] = {}
         self.environment = kwargs.pop("data_environment", DataEnvironment.MSSentinel)
         self.kql_cloud, self.az_cloud = self._set_kql_cloud()

msticpy/data/drivers/kql_driver.py

@@ -1,8 +1,8 @@
 metadata:
   version: 1
   description: MDATP Queries
-  data_environments: [MDATP, MDE, M365D]
-  data_families: [MDATP]
+  data_environments: [MDATP, MDE, M365D, LogAnalytics]
+  data_families: [MDATP, MDE]
   tags: ["file"]
 defaults:
   metadata:
@@ -22,15 +22,19 @@ defaults:
       description: Additional query clauses
       type: str
       default: ""
+    time_column:
+      description: The name of the column detailing the time the event was generated.
+      type: str
+      default: "Timestamp"
 sources:
   list_files:
     description: Lists all file events by filename
     metadata:
     args:
       query: '
         {table}
-        | where Timestamp >= datetime({start})
-        | where Timestamp <= datetime({end})
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
         | where FileName has "{file_name}"
         {add_query_items}'
       uri: None
@@ -44,8 +48,8 @@ sources:
     args:
       query: '
         {table}
-        | where Timestamp >= datetime({start})
-        | where Timestamp <= datetime({end})
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
         | where FolderPath contains "{path}"
         {add_query_items}'
     parameters:
@@ -58,8 +62,8 @@ sources:
     args:
       query: '
         {table}
-        | where Timestamp >= datetime({start})
-        | where Timestamp <= datetime({end})
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
         | where SHA1 == "{file_hash}" or SHA256 == "{file_hash}" or MD5 == "{file_hash}"
         {add_query_items}'
       uri: None