Merge branch 'main' into JV0812-patch-3

Author: JV0812

cid-redirects.json

@@ -438,6 +438,7 @@
   "/05Search/Get-Started-with-Search/Visualizations/Group-By-Operator": "/docs/search/search-query-language/search-operators",
   "/05Search/Live-Tail": "/docs/search/live-tail",
   "/05Search/Live-Tail/About-Live-Tail": "/docs/search/live-tail/about-live-tail",
+  "/Search": "/docs/search",
   "/Search/Anomaly_Detection": "/docs/alerts/monitors/create-monitor",
   "/Search/Live-Tail": "/docs/search/live-tail/about-live-tail",
   "/Search/Live-Tail/About-Live-Tail": "/docs/search/live-tail/about-live-tail",
@@ -489,6 +490,7 @@
   "/05Search/Search-Query-Language/01-Parse-Operators/04-Parse-Keyvalue-Formatted-Logs": "/docs/search/search-query-language/parse-operators/parse-keyvalue-formatted-logs",
   "/05Search/Search-Query-Language/01-Parse-Operators/05-Parse-CSV-Formatted-Logs": "/docs/search/search-query-language/parse-operators/parse-csv-formatted-logs",
   "/05Search/Search-Query-Language/01-Parse-Operators/06-Parse-Delimited-Logs-Using-Split": "/docs/search/search-query-language/parse-operators/parse-delimited-logs-using-split",
+  "/docs/search/search-query-language/parse-operators/parse-date": "/docs/search/search-query-language/parse-operators/parsedate",
   "/05Search/Search-Query-Language/01-Parse-Operators/07-Parse-XML-Formatted-Logs": "/docs/search/search-query-language/parse-operators/parse-xml-formatted-logs",
   "/05Search/Search-Query-Language/01-Parse-Operators/Parse-field-option": "/docs/search/search-query-language/parse-operators/parse-field-option",
   "/05Search/Search-Query-Language/01-Parse-Operators/Parse-nodrop-option": "/docs/search/search-query-language/parse-operators/parse-nodrop-option",
@@ -3039,6 +3041,7 @@
   "/Knowledge_Base/Parsing/Using_line_breaks_as_an_anchor_within_parse": "/docs/search/search-query-language/parse-operators/parse-predictable-patterns-using-an-anchor",
   "/Knowledge_Base/Search": "/docs/search",
   "/Knowledge_Base/Search/How_to_Prevent_your_Scheduled_Search_from_Timing_Out": "/docs/alerts/scheduled-searches/faq",
+  "/Limited_Availability": "/docs/manage/manage-subscription/beta-opt-in",
   "/Limited_Availability/Lookup_Tables": "/docs/search/search-query-language/search-operators/lookupcontains",
   "/Limited_Availability/Lookup_Tables/lookupContains_Operator": "/docs/search/search-query-language/search-operators/lookupcontains",
   "/Manage": "/docs/manage",
@@ -3381,6 +3384,7 @@
   "/Metrics/Metric-Queries-and-Alerts/10Share_a_Metric_Query": "/docs/metrics/metrics-queries/share-metric-query",
   "/Metrics/Metric-Queries-and-Alerts/11Metrics-Queries": "/docs/metrics/metrics-queries",
   "/Metrics/Metrics_Transformation_Rules": "/docs/metrics/metrics-transformation-rules",
+  "/Metrics/Understand_and_Manage_Metric_Volume/Blacklisted_Metrics_Sources": "/docs/metrics/manage-metric-volume/disabled-metrics-sources",
   "/Metrics/Understand_and_Manage_Metric_Volume/Disabled_Metrics_Sources": "/docs/metrics/manage-metric-volume/disabled-metrics-sources",
   "/Metrics/Understand_and_Manage_Metric_Volume": "/docs/metrics/manage-metric-volume",
   "/Metrics/Understand_and_Manage_Metric_Volume/Data_Limits_for_Metrics": "/docs/metrics/manage-metric-volume",
@@ -3577,6 +3581,7 @@
   "/Search/Search_Cheat_Sheets/Search_Operators_Cheat_Sheet": "/docs/search/search-cheat-sheets",
   "/Search/Search_Job_API/Search_Job_API": "/docs/api/search-job",
   "/Search/Search_Optimization": "/docs/search/optimize-search-performance",
+  "/Search/Search_Optimization/Scheduled_Views": "/docs/manage/scheduled-views",
   "/Solutions/AWS_Observability_Solution": "/docs/observability",
   "/Send_Data/Sources/03Use_JSON_to_Configure_Sources": "/docs/send-data/use-json-configure-sources",
   "/Send_Data/Sources/03Use_JSON_to_Configure_Sources/JSON_Parameters_for_Hosted_Sources": "/docs/send-data/use-json-configure-sources/json-parameters-hosted-sources",
@@ -3835,6 +3840,7 @@
   "/Beta/Cloud-to-Cloud_Integration_Framework/Workday_Source": "/docs/send-data/hosted-collectors/cloud-to-cloud-integration-framework/workday-source",
   "/Beta/Dashboard-Data-API": "/docs/api/dashboard",
   "/Beta/Dashboard_(New)": "/docs/dashboards",
+  "/Beta/Dashboard_(Beta)/01Sumo_Logic's_New_Dashboard_(Beta)": "/docs/dashboards",
   "/Beta/Dashboard_(Beta)/Create_a_New_Dashboard_(Beta)": "/docs/dashboards",
   "/Beta/Fields": "/docs/manage/fields",
   "/Beta/Grant_Access_to_Data_in_Audit_Indexes": "/docs/manage/users-roles/roles/create-manage-roles",
@@ -3973,6 +3979,7 @@
   "/Send_Data/Data_Types/Host_Metrics/02Install_the_Host_Metrics_App": "/docs/integrations/hosts-operating-systems/host-metrics",
   "/Send_Data/Data_Types/Linux/Collect_Logs_for_Linux": "/docs/integrations/hosts-operating-systems/linux",
   "/Send_Data/Data_Types/Observable_Networks/Observable_Networks_App_Dashboard_and_Searches": "/docs/integrations/security-threat-detection/observable-networks",
+  "/Send_Data/Data_Types/OneLogin/Collect_Logs_for_OneLogin": "/docs/integrations/saml/onelogin",
   "/Send_Data/Data_Types/PagerDuty/PagerDuty_App_Dashboards": "/docs/integrations/saas-cloud/pagerduty-v3",
   "/Send_Data/Data_Types/Threat_Intel_for_AWS/Threat_Intel_for_AWS_App_Dashboard": "/docs/integrations/amazon-aws/threat-intel",
   "/Send_Data/Installed_Collectors": "/docs/send-data/installed-collectors",
@@ -4038,6 +4045,7 @@
   "/Send-Data/Applications-and-Other-Data-Sources/Okta/Collect-Okta-Logs": "/docs/integrations/saml/okta",
   "/Send-Data/Applications-and-Other-Data-Sources/Oracle/00Collect_Logs_for_Oracle": "/docs/send-data/collect-from-other-data-sources/collect-logs-oracle-cloud-infrastructure",
   "/Send-Data/Applications-and-Other-Data-Sources/Palo_Alto_Networks_8/Collect_Logs_for_Palo_Alto_Networks_8": "/docs/integrations/security-threat-detection/palo-alto-networks-9",
+  "/Send-Data/Applications-and-Other-Data-Sources/Puppet": "/docs/integrations/app-development/puppet",
   "/Send-Data/Applications-and-Other-Data-Sources/Threat-Intel-Quick-Analysis": "/docs/integrations/security-threat-detection/threat-intel-quick-analysis",
   "/Send-Data/Applications-and-Other-Data-Sources/Threat-Intel-Quick-Analysis/Threat-Intel-FAQ": "/docs/integrations/security-threat-detection/threat-intel-quick-analysis",
   "/Send-Data/Applications-and-Other-Data-Types/Okta": "/docs/integrations/saml/okta",
@@ -4210,6 +4218,7 @@
   "/docs/dashboards/edit-dashboards": "/docs/dashboards",
   "/docs/dashboards/chart-panel-types/numerical-single-value-charts": "/docs/dashboards/panels/single-value-charts",
   "/docs/dashboards/edit-dashboards/manage-axis": "/docs/dashboards",
+  "/docs/dashboards/edit-dashboards/manage-charts": "/docs/dashboards/panels",
   "/docs/dashboards/edit-dashboards/manage-dashboards": "/docs/dashboards",
   "/docs/dashboards/chart-panel-types/box-plot-charts": "/docs/dashboards/panels/box-plot-charts",
   "/docs/dashboards/chart-panel-types/line-charts": "/docs/dashboards/panels/line-charts",
@@ -4308,6 +4317,7 @@
   "/docs/manage/partitions/flex/flex-pricing-faqs": "/docs/manage/partitions/flex/faq",
   "/docs/manage/partitions/flex/flex-pricing-faq": "/docs/manage/partitions/flex/faq",
   "/docs/platform-services/automation-service/app-central/integrations/exana-open-dns": "/docs/platform-services/automation-service/app-central/integrations",
+  "/docs/platform-services/automation-service/app-central/integrations/malwarebytes-oneview": "/docs/platform-services/automation-service/app-central/integrations/malwarebytes-nebula",
   "/docs/platform-services/automation-service/app-central/integrations/snowflake": "/docs/platform-services/automation-service/app-central/integrations",
   "/docs/integrations/security-threat-detection/palo-alto-networks-6": "/docs/integrations/security-threat-detection/palo-alto-networks-9",
   "/docs/integrations/security-threat-detection/palo-alto-networks-8":"/docs/integrations/security-threat-detection/palo-alto-networks-9",

docs/manage/ingestion-volume/log-ingestion.md

@@ -29,7 +29,7 @@ Log data may not be kept when sent via HTTP Sources or Cloud Syslog Sources, as
 * Sumo Logic accounts can be upgraded at any time to allow for additional quota. Contact [Sumo Logic Sales](mailto:[email protected]) to customize your account to meet your organization's needs.
 
 :::important
-Compressed files are decompressed before they are ingested, so they are ingested at the decompressed file size rate.
+[Compressed files](/docs/send-data/hosted-collectors/http-source/logs-metrics/#compressed-data) are decompressed before they are ingested, so they are ingested at the decompressed file size rate.
 :::
 
 ## Log Throttling

docs/search/optimize-search-performance.md

@@ -70,3 +70,191 @@ Here's a quick look at how to choose the right indexed search optimization tool.
 As data enters Sumo Logic, it is first routed to any Partitions for indexing. It is then checked against Scheduled Views, and any data that matches the Scheduled Views is indexed.
 
 Data can be in both a Partition and a Scheduled View because the two tools are used differently (and are indexed separately). Although Partitions are indexed first, the process does not slow the indexing of Scheduled Views.
+
+## Additional methods to optimize Search performance
+
+### Use the smallest time range
+
+Always set the search time range to the minimum duration required for your use case. This reduces the data volume and improves the query efficiency. When working with long time ranges, start by building and testing your search on a shorter time range. Once the search is finalized and validated, extend it to cover the entire period needed for your analysis. 
+
+### Use fields extracted by FERs
+
+Instead of relying on the `where` operator, filter the data using fields that are already extracted through the Field Extraction Rules (FERs) in the source expression. This approach is more efficient and improves query performance.
+
+**Not recommended approach:**
+
+```
+_sourceCategory=foo
+| where field_a="value_a"
+```
+
+**Recommended approach:**
+
+```
+sourceCategory=foo and field_a=value_a
+```
+
+### Move terms from parse statement to source expression
+
+Adding the parsing terms in the source expression will help you enhance the search performance. A parse statement without `nodrop` drops the logs that could not parse the desired field. For example, `parse "completed * action" as actionName` will remove logs that do not have **completed** and **action** terms. 
+
+**Not recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "completed * action" as actionName
+| count by actionName
+```
+
+**Recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog completed action
+| parse "completed * action" as actionName
+| count by actionName
+```
+
+### Filter data before aggregation
+
+While filtering the date, reduce the result set to the smallest possible size before performing aggregate operations such as sum, min, max, and average. Also, use subquery in source expression instead of using `if` or `where` search operators. 
+
+**Not recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "userName: *, " as user
+| count by user
+| where user="john"
+```
+
+**Recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog userName
+| parse "userName: *, " as user
+| where user="john"
+| count by user
+```
+
+### Remove redundant operators
+
+Remove the search operators in the query that are not required for the desired results. 
+
+For example, let’s say you have a `sort` operator before an aggregation, but this sorting does not make any difference to the aggregated results. 
+
+**Not recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "userName: *, " as user
+| parse "evenName: *, " as event
+| count by user
+```
+
+**Recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "userName: *, " as user
+| count by user
+```
+
+### Merge operators
+
+If the same operators are used multiple times in different levels of query, if possible, try to merge these similar operators. Also, do not use the same operator multiple times to get the same value. This helps in reducing the number of passes performed on the data thereby improving the search performance.
+
+**Example 1:**
+
+    **Not recommended approach:**
+
+    ```
+    _sourceCategory=Prod/User/Eventlog
+    | parse "completed * action" as actionName
+    | parse "action in * ms" as duration
+    | pct(duration, 95) by actionName
+    ```
+
+    **Recommended approach:**
+
+    ```
+    _sourceCategory=Prod/User/Eventlog
+    | parse "completed * action in * ms" as actionName, duration
+    | pct(duration, 95) by actionName
+    ```
+
+**Example 2:**
+
+    **Not recommended approach:**
+
+    ```
+    _sourceCategory=Prod/User/Eventlog
+    | parse "completed * action" as actionName
+    | where toLowerCase(actionName) = "logIn” or toLowerCase(actionName) matches "abc*” or toLowerCase(actionName) contains "xyz"
+    ```
+
+    **Recommended approach:**
+
+    ```
+    _sourceCategory=Prod/User/Eventlog
+    | parse "completed * action" as actionName
+    | toLowerCase(actionName) as actionNameLowered
+    | where actionNameLowered = "logIn” or actionNameLowered matches "abc*” or actionNameLowered contains "xyz”
+    ```
+
+### Use lookup on the lowest possible dataset
+
+Minimize the data processed by the `lookup` operator in the query, as lookup is an expensive operation. It can be done in two ways:
+
+- Use the lookup as late as possible in the query assuming that clauses before lookup are doing additional data filtering.
+- Move the lookup after an aggregation to drastically reduce the data processed by lookup, as aggregated data is generally far less than non-aggregated data.
+
+**Not recommended approach:**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "completed * action in * ms" as actionName, duration
+| lookup actionType from path://"/Library/Users/[email protected]/actionTypes" on actionName
+| where actionName in ("login”, "logout”)
+| count by actionName, actionType
+```
+
+**Recommended approach (Option 1):**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "completed * action in * ms" as actionName, duration
+| where actionName in ("login”, "logout”)
+| count by actionName
+| lookup actionType from path://"/Library/Users/[email protected]/actionTypes" on actionName
+```
+
+**Recommended approach (Option 2):**
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse "completed * action in * ms" as actionName, duration
+| where actionName in ("login”, "logout”)
+| lookup actionType from path://"/Library/Users/[email protected]/actionTypes" on actionName
+| count by actionName, actionType 
+```
+
+### Avoid multiple parse multi statements
+
+A parse `multi` statement causes a single log to produce multiple logs in the results. But if a parse `multi` statement is followed by more parse `multi` statements, it can lead to data explosion and the query may never finish. Even if the query works the results may not be as expected.
+
+For example, consider the below query where the assumption is that a single log line contains multiple users and multiple event names.
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse regex "userName: (?<user>[a-z-A-Z]+), " multi
+| parse regex "eventName: (?<event>[a-z-A-Z]+), " multi
+```
+
+But if you write the query like that, it will generate a result for every combination of `userName` and `eventName` values. Now suppose you want to count by `eventName`, it will not give you the desired result, since a single `eventName` has been duplicated for every `userName` in the same log. So, the better query would be:
+
+```
+_sourceCategory=Prod/User/Eventlog
+| parse regex "userName: (?<user>[a-z-A-Z]+), eventName: (?<event>[a-z-A-Z]+), " multi
+```
+
+

docs/send-data/collector-faq.md

@@ -419,9 +419,15 @@ This article describes the assumptions that Sumo makes about customer data, tips
 
 See [using _format for troubleshooting](/docs/send-data/reference-information/time-reference.md) timestamps.
 
-#### Assumption: Data is less than 365 days old
+#### Assumption: Data is less than 30 days but within 365 days
 
-Sumo Logic assumes that all log message times fall within a window of -1 year through +2 days compared to the current time. Any log messages with a parsed timestamp outside of that window is automatically re-stamped with the current time.
+* To ingest historical data older than 30 days but within 365 days, you must specify a `timestamp` field using a regex locator and a valid date format.
+
+#### Assumption: Data is older than 365 days
+
+Sumo Logic assumes that all log message times fall within a window of -1 year through +2 days compared to the current time. Any log messages with a parsed timestamp outside of that window are automatically re-stamped with the current time.
+* Data older than 365 days can still be ingested. However, even if a custom timestamp is provided, it will be autocorrected to the current time unless technical support disables this function at the organization level.
+* To ingest data older than 365 days with the original timestamp intact, you'll need to contact [Support](https://support.sumologic.com/support/s) to disable the autocorrection function at the org level.
 
 #### Assumption: Data from a source will have similar timestamps
 

docs/send-data/reference-information/time-reference.md

@@ -33,7 +33,7 @@ If your log messages from a Source contain multiple timestamps, timestamps in un
 
 The Collector assumes that all log messages coming from a particular Source will have timestamps that are close together. If a message comes through that appears to be more than one day earlier or later than recent messages from that Source it will be auto-corrected to match the current time. You can stop this auto-correction by explicitly configuring a custom timestamp format on your Source.
 
-The Collector also assumes that all log messages coming from a particular Source will have timestamps that are within a window of -1 year through +2 days compared to the current time. Any log message with a parsed timestamp outside of that window is automatically re-stamped with the current time. You must contact [Sumo Logic Support](https://support.sumologic.com/) to adjust this auto-correction behavior. See [How to ingest old or historical data](/docs/send-data/collector-faq#how-to-ingest-old-or-historical-data) for further details.
+The Collector also assumes that all log messages coming from a particular Source will have timestamps that are within a window of -1 year through +2 days compared to the current time. Any log messages with a timestamp older than 30 days is automatically set to the current time. You must contact [Sumo Logic Support](https://support.sumologic.com/) to adjust this auto-correction behavior. See [How to ingest old or historical data](/docs/send-data/collector-faq#how-to-ingest-old-or-historical-data) for further details.
 
 ### Automated timestamp parsing