diff --git a/data-explorer/kusto-tocs/management/toc.yml b/data-explorer/kusto-tocs/management/toc.yml index 6acb387350..2744121283 100644 --- a/data-explorer/kusto-tocs/management/toc.yml +++ b/data-explorer/kusto-tocs/management/toc.yml @@ -849,6 +849,23 @@ items: - name: .ingest inline command displayName: .ingest inline href: /kusto/management/data-ingestion/ingest-inline?view=azure-data-explorer&preserve-view=true + - name: Queued ingestion commands + items: + - name: Queued ingestion commands overview + displayName: Queued ingestion commands, ingest from folders, ingest from files, ingest from storage + href: /kusto/management/data-ingestion/queued-ingestion-overview?view=azure-data-explorer&preserve-view=true + - name: Queued ingestion commands use case + displayName: Queued ingestion historical data ingestion + href: /kusto/management/data-ingestion/queued-ingestion-use-case?view=azure-data-explorer&preserve-view=true + - name: .ingest-from-storage-queued command + displayName: .ingest-from-storage-queued, ingest from storage command + href: /kusto/management/data-ingestion/ingest-from-storage-queued?view=azure-data-explorer&preserve-view=true + - name: .list blobs command + href: /kusto/management/data-ingestion/list-blobs?view=azure-data-explorer&preserve-view=true + - name: .cancel queued ingestion operation command + href: /kusto/management/data-ingestion/cancel-queued-ingestion-operation-command?view=azure-data-explorer&preserve-view=true + - name: .show queued ingestion operation command + href: /kusto/management/data-ingestion/show-queued-ingestion-operations?view=azure-data-explorer&preserve-view=true - name: Streaming ingestion items: - name: Streaming ingestion and schema changes diff --git a/data-explorer/kusto-tocs/query/toc.yml b/data-explorer/kusto-tocs/query/toc.yml index 4fa9fecca2..e81d85a7da 100644 --- a/data-explorer/kusto-tocs/query/toc.yml +++ b/data-explorer/kusto-tocs/query/toc.yml @@ -116,10 +116,13 @@ items: displayName: functions library, graph, blast radius, cyber, security, cybersecurity href: /kusto/functions-library/graph-blast-radius-fl?view=azure-data-explorer&preserve-view=true - name: graph_exposure_perimeter_fl() - displayName: functions library, graph, exposure, perimeter, cyber, security, cybersecurity + displayName: functions library, graph, exposure perimeter, cyber, security, cybersecurity href: /kusto/functions-library/graph-exposure-perimeter-fl?view=azure-data-explorer&preserve-view=true + - name: graph_node_centrality_fl() + displayName: functions library, graph, centrality, cyber, security, cybersecurity + href: /kusto/functions-library/graph-node-centrality-fl?view=azure-data-explorer&preserve-view=true - name: graph_path_discovery_fl() - displayName: functions library, graph, path, discovery, cyber, security, cybersecurity, + displayName: functions library, graph, path discovery, cyber, security, cybersecurity href: /kusto/functions-library/graph-path-discovery-fl?view=azure-data-explorer&preserve-view=true - name: kmeans_fl() displayName: functions library, clustering, K-Means @@ -1381,6 +1384,10 @@ items: - name: countif() displayName: count(predicate) href: /kusto/query/countif-aggregation-function?view=azure-data-explorer&preserve-view=true + - name: covariance() + href: /kusto/query/covariance-aggregation-function?view=azure-data-explorer&preserve-view=true + - name: covariancep() + href: /kusto/query/covariancep-aggregation-function?view=azure-data-explorer&preserve-view=true - name: dcount() href: /kusto/query/dcount-aggregation-function?view=azure-data-explorer&preserve-view=true - name: dcountif() @@ -1491,21 +1498,29 @@ items: href: /kusto/query/node-degree-out?view=azure-data-explorer&preserve-view=true - name: Geospatial items: - - name: Geospatial clustering overview + - name: Geospatial clustering displayName: H3, S2, Geohash href: /kusto/query/geospatial-grid-systems?view=azure-data-explorer&preserve-view=true + - name: Geospatial joins + href: /kusto/query/geospatial-joins?view=azure-data-explorer&preserve-view=true - name: Geospatial visualizations href: /kusto/query/geospatial-visualizations?view=azure-data-explorer&preserve-view=true - name: geo_angle() href: /kusto/query/geo-angle-function?view=azure-data-explorer&preserve-view=true - name: geo_azimuth() href: /kusto/query/geo-azimuth-function?view=azure-data-explorer&preserve-view=true + - name: geo_closest_point_on_line() + href: /kusto/query/geo-closest-point-on-line-function?view=azure-data-explorer&preserve-view=true + - name: geo_closest_point_on_polygon() + href: /kusto/query/geo-closest-point-on-polygon-function?view=azure-data-explorer&preserve-view=true - name: geo_distance_2points() href: /kusto/query/geo-distance-2points-function?view=azure-data-explorer&preserve-view=true - name: geo_distance_point_to_line() href: /kusto/query/geo-distance-point-to-line-function?view=azure-data-explorer&preserve-view=true - name: geo_distance_point_to_polygon() href: /kusto/query/geo-distance-point-to-polygon-function?view=azure-data-explorer&preserve-view=true + - name: geo_from_wkt() + href: /kusto/query/geo-from-wkt-function?view=azure-data-explorer&preserve-view=true - name: geo_intersects_2lines() href: /kusto/query/geo-intersects-2lines-function?view=azure-data-explorer&preserve-view=true - name: geo_intersects_2polygons() @@ -1524,8 +1539,12 @@ items: href: /kusto/query/geo-line-centroid-function?view=azure-data-explorer&preserve-view=true - name: geo_line_densify() href: /kusto/query/geo-line-densify-function?view=azure-data-explorer&preserve-view=true + - name: geo_line_interpolate_point() + href: /kusto/query/geo-line-interpolate-point-function?view=azure-data-explorer&preserve-view=true - name: geo_line_length() href: /kusto/query/geo-line-length-function?view=azure-data-explorer&preserve-view=true + - name: geo_line_locate_point() + href: /kusto/query/geo-line-locate-point-function?view=azure-data-explorer&preserve-view=true - name: geo_line_simplify() href: /kusto/query/geo-line-simplify-function?view=azure-data-explorer&preserve-view=true - name: geo_line_to_s2cells() diff --git a/data-explorer/kusto/query/best-practices.md b/data-explorer/kusto/query/best-practices.md index 1ab5df839f..65c35cef2e 100644 --- a/data-explorer/kusto/query/best-practices.md +++ b/data-explorer/kusto/query/best-practices.md @@ -72,8 +72,8 @@ In order of importance: * Apply the `where` query operator immediately following table references. * When using the `where` query operator, the order in which you place the predicates, whether you use a single `where` operator, or multiple consecutive `where` operators, - can have a significant effect on the query performance. - + can have a significant effect on the query performance, In many cases, the query optimizer will automatically arrange the predicates in an efficient order. However, this is not always guaranteed—so when it doesn't, you should manually order the predicates according to the guidelines in the next points. + * Apply predicates that act upon `datetime` table columns first. Kusto includes an efficient index on such columns, often completely eliminating whole data shards without needing to access those shards. diff --git a/data-explorer/kusto/query/reduce-operator.md b/data-explorer/kusto/query/reduce-operator.md index 7d2dc78166..0e76089ced 100644 --- a/data-explorer/kusto/query/reduce-operator.md +++ b/data-explorer/kusto/query/reduce-operator.md @@ -24,8 +24,8 @@ For each such group, the operator returns a `pattern`, `count`, and `representat | Name | Type | Required | Description | |--|--|--|--| | *Expr* | `string` | :heavy_check_mark: | The value by which to reduce.| -| *Threshold* | `real` | | A value between 0 and 1 that determines the minimum fraction of rows required to match the grouping criteria in order to trigger a reduction operation. The default value is 0.1.

We recommend setting a small threshold value for large inputs. With a smaller threshold value, more similar values are grouped together, resulting in fewer but more similar groups. A larger threshold value requires less similarity, resulting in more groups that are less similar. See [Examples](#examples).| -| *Characters* | `string` | | A list of characters that separate between terms. The default is every non-ascii numeric character. For examples, see [Behavior of Characters parameter](#behavior-of-characters-parameter).| +| *Threshold* | `real` | | A value between 0 and 1 that determines the minimum fraction of rows required to match the grouping criteria in order to trigger a reduction operation. The default value is 0.1.

The threshold parameter determines the minimum level of similarity required for values to be grouped together. With a smaller threshold value (closer to 0), more similar values are grouped together, resulting in fewer but more similar groups. A larger threshold value (closer to 1) requires less similarity, resulting in more groups that are less similar. We recommend setting a small threshold value for large inputs. See [Examples](#examples).| +| *Characters* | `string` | | A list of characters that separate between terms. The default is every non-ascii numeric character. For examples, see [Examples](#examples).| | *ReduceKind* | `string` | | The only valid value is `source`. If `source` is specified, the operator appends the `Pattern` column to the existing rows in the table instead of aggregating by `Pattern`.| ## Returns @@ -45,12 +45,11 @@ For example, the result of `reduce by city` might include: ## Examples The example in this section shows how to use the syntax to help you get started. - -[!INCLUDE [help-cluster](../includes/help-cluster-note.md)] -### Small threshold value +[!INCLUDE [help-cluster](../includes/help-cluster-note.md)] -This query generates a range of numbers, creates a new column with concatenated strings and random integers, and then groups the rows by the new column with specific reduction parameters. +The following example generates a range of numbers, creates a new column with concatenated strings and random integers, and then groups the rows by the new column with specific reduction parameters. +The threshold is set to 0.001, which means that the operator groups values that are very similar to each other. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] @@ -69,9 +68,8 @@ range x from 1 to 1000 step 1 |----------------|-----|-----------------| |MachineLearning*|1000 |MachineLearningX4| -### Large threshold value - -This query generates a range of numbers, creates a new column with concatenated strings and random integers, and then groups the rows by the new column with specific reduction parameters. +The following example generates a range of numbers, creates a new column with concatenated strings and random integers, and then groups the rows by the new column with specific reduction parameters. +The threshold is set to 0.9, which means that the operator groups together values less strictly and allows more variance. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] @@ -100,9 +98,8 @@ The result includes only those groups where the MyText value appears in at least | MachineLearning* | 104 | MachineLearningX7 | | MachineLearning* | 106 | MachineLearningX2 | -### Behavior of `Characters` parameter - If the *Characters* parameter is unspecified, by default the operator treats all non-alphanumeric characters (including spaces and punctuation) as term separators. +The following example shows how the `reduce` operator behaves when the *Characters* parameter isn't specified. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] @@ -136,17 +133,13 @@ range x from 1 to 10 step 1 | project str = strcat("foo", "Z", tostring(x)) | re |--|--|--| |foo*|10|fooZ1| -### Apply `reduce` to sanitized input - The following example shows how one might apply the `reduce` operator to a "sanitized" input, in which GUIDs in the column being reduced are replaced before reducing: Start with a few records from the Trace table. Then reduce the Text column which includes random GUIDs. -As random GUIDs interfere with the reduce operation, replace them all -by the string "GUID". -Now perform the reduce operation. In case there are other "quasi-random" identifiers with embedded '-' -or '_' characters in them, treat characters as non-term-breakers. +As random GUIDs interfere with the reduce operation, replace them all by the string "GUID". +Now perform the reduce operation. In case there are other "quasi-random" identifiers with embedded '-' or '_' characters in them, treat characters as non-term-breakers. ```kusto Trace