Merge pull request #2855 from pnp/search-improvments

erwinvanhunen · web-flow · commit 0f894248a858 · 2020-08-20T10:03:16.000+02:00
Improve -All queries.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed issue with Submit-PnPTeamsChannelMessage not posting HTML message when setting the content type to Html.
 - The content type that Submit-PnPTeamsChannelMessage uses defaults now to HTML.
 - Fixed an issue with the PnP Provisioning Engine not being able to correctly acquire a token for the Microsoft Graph when provisioning a tenant template containing a Team.
+- Optimized Submit-PnPSearchQuery and Get-PnPSiteSearchQueryResults cmdlets when using the -All parameter.
+- Fixed TrimDuplicates to be default off for Submit-PnPSearchQuery
 
 ### Contributors
 
diff --git a/Commands/Search/GetSiteSearchQueryResults.cs b/Commands/Search/GetSiteSearchQueryResults.cs
@@ -1,9 +1,6 @@
 ﻿using System.Management.Automation;
-using Microsoft.SharePoint.Client;
-using Microsoft.SharePoint.Client.Search.Query;
 using PnP.PowerShell.CmdletHelpAttributes;
 using System.Collections.Generic;
-using System.Linq;
 
 namespace PnP.PowerShell.Commands.Search
 {
@@ -52,68 +49,40 @@ public class GetSiteSearchQueryResults : PnPWebCmdlet
 
         protected override void ExecuteCmdlet()
         {
-            int startRow = StartRow;
-            int rowLimit = MaxResults;
-            if (All.IsPresent)
+            var queryCmdLet = new SubmitSearchQuery();
+            queryCmdLet.StartRow = StartRow;
+            queryCmdLet.MaxResults = MaxResults;
+            var query = "contentclass:STS_Site";
+            if (!string.IsNullOrEmpty(Query))
             {
-                startRow = 0;
-                rowLimit = 500;
+                query = query + " AND " + Query;
             }
-            int currentCount = 0;
-            var dynamicList = new List<dynamic>();
-            do
-            {
-                var keywordQuery = GetKeywordQuery();
-                keywordQuery.StartRow = startRow;
-                keywordQuery.RowLimit = rowLimit;
+            queryCmdLet.Query = query;
 
-                var searchExec = new SearchExecutor(ClientContext);
-                var results = searchExec.ExecuteQuery(keywordQuery);
-                ClientContext.ExecuteQueryRetry();
-
-                if (results?.Value[0].RowCount > 0)
-                {
-                    var result = results.Value[0];
-                    currentCount = result.ResultRows.Count();
-
-                    foreach (var row in result.ResultRows)
-                    {
-                        dynamicList.Add(
-                            new
-                            {
-                                Title = row["Title"]?.ToString() ?? "",
-                                Url = row["SPSiteUrl"]?.ToString() ?? "",
-                                Description = row["Description"]?.ToString() ?? "",
-                                WebTemplate = row["WebTemplate"]?.ToString() ?? ""
-                            });
-                    }
-                }
-                startRow += rowLimit;
-
-            } while (currentCount == rowLimit && All.IsPresent);
-            WriteObject(dynamicList, true);
-        }
+            queryCmdLet.SelectProperties = new[] {"Title","SPSiteUrl","Description","WebTemplate"};
+            queryCmdLet.SortList = new System.Collections.Hashtable
+            {
+                { "SPSiteUrl", "ascending" }
+            };
+            queryCmdLet.RelevantResults = true;
 
-        private KeywordQuery GetKeywordQuery()
-        {
-            var keywordQuery = new KeywordQuery(ClientContext);
+            var res = queryCmdLet.Run();
 
-            // Construct query to execute
-            var query = "contentclass:STS_Site";
-            if (!string.IsNullOrEmpty(Query))
+            var dynamicList = new List<dynamic>();
+            foreach (var row in res)
             {
-                query = query + " AND " + Query;
+                var obj = row as PSObject;                
+                dynamicList.Add(
+                    new
+                    {
+                        Title = obj.Properties["Title"]?.Value ?? "",
+                        Url = obj.Properties["SPSiteUrl"]?.Value ?? "",
+                        Description = obj.Properties["Description"]?.Value ?? "",
+                        WebTemplate = obj.Properties["WebTemplate"]?.Value ?? ""
+                    });
             }
 
-            keywordQuery.QueryText = query;
-            keywordQuery.SelectProperties.Add("Title");
-            keywordQuery.SelectProperties.Add("SPSiteUrl");
-            keywordQuery.SelectProperties.Add("Description");
-            keywordQuery.SelectProperties.Add("WebTemplate");
-            keywordQuery.SortList.Add("SPSiteUrl", SortDirection.Ascending);
-            // Important to avoid trimming "similar" site collections
-            keywordQuery.TrimDuplicates = false;
-            return keywordQuery;
+            WriteObject(dynamicList, true);
         }
     }
 }
diff --git a/Commands/Search/SubmitSearchQuery.cs b/Commands/Search/SubmitSearchQuery.cs
@@ -105,34 +105,43 @@ public class SubmitSearchQuery : PnPWebCmdlet
         [Parameter(Mandatory = false, HelpMessage = "Specifies whether only relevant results are returned", ParameterSetName = ParameterAttribute.AllParameterSets)]
         public SwitchParameter RelevantResults;
 
-        protected override void ExecuteCmdlet()
+        internal IEnumerable<object> Run()
         {
             int startRow = StartRow;
             int rowLimit = MaxResults;
+            // Used for internal Microsoft telemetry
+            string clientFunction = "Search";
             if (All.IsPresent)
             {
+                clientFunction = "Scanner";
                 startRow = 0;
                 rowLimit = 500;
                 SortList = new Hashtable()
                     {
-                        {"IndexDocId", "ascending"}
+                        {"[DocId]", "ascending"}    // Special optimized sorting when iterating all items
                     };
             }
 
             var currentCount = 0;
+            string lastDocId = "0";
             PnPResultTableCollection finalResults = null;
             do
             {
-                KeywordQuery keywordQuery = CreateKeywordQuery();
+                KeywordQuery keywordQuery = CreateKeywordQuery(clientFunction);
                 keywordQuery.StartRow = startRow;
                 keywordQuery.RowLimit = rowLimit;
 
-                var searchExec = new SearchExecutor(ClientContext);
-                if (startRow > 0 && All.IsPresent)
+                if (All.IsPresent)
                 {
-                    keywordQuery.Refiners = null; // Only need to set on first page for auto paging
+                    if (currentCount != 0)
+                    {
+                        keywordQuery.Refiners = null; // Only need to set on first page for auto paging
+                    }
+                    keywordQuery.StartRow = 0;
+                    keywordQuery.QueryText += " IndexDocId>" + lastDocId;
                 }
 
+                var searchExec = new SearchExecutor(ClientContext);
                 var results = searchExec.ExecuteQuery(keywordQuery);
                 ClientContext.ExecuteQueryRetry();
 
@@ -146,6 +155,7 @@ protected override void ExecuteCmdlet()
                             if (resultTable.TableType == "RelevantResults")
                             {
                                 currentCount = resultTable.RowCount;
+                                lastDocId = resultTable.ResultRows.Last()["DocId"].ToString();
                             }
                         }
                     }
@@ -167,25 +177,36 @@ protected override void ExecuteCmdlet()
                             if (pnpResultTable.TableType == "RelevantResults")
                             {
                                 currentCount = resultTable.RowCount;
+                                if (currentCount > 0)
+                                {
+                                    lastDocId = resultTable.ResultRows.Last()["DocId"].ToString();
+                                }
                             }
                         }
                     }
 
                 }
                 startRow += rowLimit;
             } while (currentCount == rowLimit && All.IsPresent);
+
             if (!RelevantResults.IsPresent)
             {
-                WriteObject(finalResults, true);
+                return finalResults;
             }
             else
             {
                 var results = finalResults.FirstOrDefault(t => t.TableType == "RelevantResults")?
                     .ResultRows.Select(r => ConvertToPSObject(r));
-                WriteObject(results, true);
+                return results;
             }
         }
 
+        protected override void ExecuteCmdlet()
+        {
+            var results = Run();
+            WriteObject(results, true);
+        }
+
         private object ConvertToPSObject(IDictionary<string, object> r)
         {
             PSObject res = new PSObject();
@@ -199,7 +220,7 @@ private object ConvertToPSObject(IDictionary<string, object> r)
             return res;
         }
 
-        private KeywordQuery CreateKeywordQuery()
+        private KeywordQuery CreateKeywordQuery(string clientFunction)
         {
             var keywordQuery = new KeywordQuery(ClientContext);
 
@@ -212,7 +233,7 @@ private KeywordQuery CreateKeywordQuery()
 
             keywordQuery.QueryText = query;
             keywordQuery.ClientType = ClientType;
-            if (ParameterSpecified(nameof(TrimDuplicates))) keywordQuery.TrimDuplicates = TrimDuplicates;
+            keywordQuery.TrimDuplicates = TrimDuplicates;
             if (ParameterSpecified(nameof(Refiners))) keywordQuery.Refiners = Refiners;
             if (ParameterSpecified(nameof(Culture))) keywordQuery.Culture = Culture;
             if (ParameterSpecified(nameof(QueryTemplate))) keywordQuery.QueryTemplate = QueryTemplate;
@@ -245,7 +266,7 @@ private KeywordQuery CreateKeywordQuery()
                 {
                     SelectProperties = SelectProperties[0].Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                 }
-                
+
                 foreach (string property in SelectProperties)
                 {
                     selectProperties.Add(property.Trim());
@@ -289,6 +310,12 @@ private KeywordQuery CreateKeywordQuery()
                     keywordQuery.Properties.SetQueryPropertyValue(key, propVal);
                 }
             }
+
+            QueryPropertyValue clientFuncPropVal = new QueryPropertyValue();
+            clientFuncPropVal.StrVal = clientFunction;
+            clientFuncPropVal.QueryPropertyValueTypeIndex = 1;
+
+            keywordQuery.Properties.SetQueryPropertyValue("ClientFunction", clientFuncPropVal);
             return keywordQuery;
         }
     }

Original file line number	Diff line number	Diff line change
`@@ -105,34 +105,43 @@ public class SubmitSearchQuery : PnPWebCmdlet`
`105`	`105`	`[Parameter(Mandatory = false, HelpMessage = "Specifies whether only relevant results are returned", ParameterSetName = ParameterAttribute.AllParameterSets)]`
`106`	`106`	`public SwitchParameter RelevantResults;`
`107`	`107`
`108`		`- protected override void ExecuteCmdlet()`
	`108`	`+ internal IEnumerable<object> Run()`
`109`	`109`	`{`
`110`	`110`	`int startRow = StartRow;`
`111`	`111`	`int rowLimit = MaxResults;`
	`112`	`+ // Used for internal Microsoft telemetry`
	`113`	`+ string clientFunction = "Search";`
`112`	`114`	`if (All.IsPresent)`
`113`	`115`	`{`
	`116`	`+ clientFunction = "Scanner";`
`114`	`117`	`startRow = 0;`
`115`	`118`	`rowLimit = 500;`
`116`	`119`	`SortList = new Hashtable()`
`117`	`120`	`{`
`118`		`- {"IndexDocId", "ascending"}`
	`121`	`+ {"[DocId]", "ascending"} // Special optimized sorting when iterating all items`
`119`	`122`	`};`
`120`	`123`	`}`
`121`	`124`
`122`	`125`	`var currentCount = 0;`
	`126`	`+ string lastDocId = "0";`
`123`	`127`	`PnPResultTableCollection finalResults = null;`
`124`	`128`	`do`
`125`	`129`	`{`
`126`		`- KeywordQuery keywordQuery = CreateKeywordQuery();`
	`130`	`+ KeywordQuery keywordQuery = CreateKeywordQuery(clientFunction);`
`127`	`131`	`keywordQuery.StartRow = startRow;`
`128`	`132`	`keywordQuery.RowLimit = rowLimit;`
`129`	`133`
`130`		`- var searchExec = new SearchExecutor(ClientContext);`
`131`		`- if (startRow > 0 && All.IsPresent)`
	`134`	`+ if (All.IsPresent)`
`132`	`135`	`{`
`133`		`- keywordQuery.Refiners = null; // Only need to set on first page for auto paging`
	`136`	`+ if (currentCount != 0)`
	`137`	`+ {`
	`138`	`+ keywordQuery.Refiners = null; // Only need to set on first page for auto paging`
	`139`	`+ }`
	`140`	`+ keywordQuery.StartRow = 0;`
	`141`	`+ keywordQuery.QueryText += " IndexDocId>" + lastDocId;`
`134`	`142`	`}`
`135`	`143`
	`144`	`+ var searchExec = new SearchExecutor(ClientContext);`
`136`	`145`	`var results = searchExec.ExecuteQuery(keywordQuery);`
`137`	`146`	`ClientContext.ExecuteQueryRetry();`
`138`	`147`
`@@ -146,6 +155,7 @@ protected override void ExecuteCmdlet()`
`146`	`155`	`if (resultTable.TableType == "RelevantResults")`
`147`	`156`	`{`
`148`	`157`	`currentCount = resultTable.RowCount;`
	`158`	`+ lastDocId = resultTable.ResultRows.Last()["DocId"].ToString();`
`149`	`159`	`}`
`150`	`160`	`}`
`151`	`161`	`}`
`@@ -167,25 +177,36 @@ protected override void ExecuteCmdlet()`
`167`	`177`	`if (pnpResultTable.TableType == "RelevantResults")`
`168`	`178`	`{`
`169`	`179`	`currentCount = resultTable.RowCount;`
	`180`	`+ if (currentCount > 0)`
	`181`	`+ {`
	`182`	`+ lastDocId = resultTable.ResultRows.Last()["DocId"].ToString();`
	`183`	`+ }`
`170`	`184`	`}`
`171`	`185`	`}`
`172`	`186`	`}`
`173`	`187`
`174`	`188`	`}`
`175`	`189`	`startRow += rowLimit;`
`176`	`190`	`} while (currentCount == rowLimit && All.IsPresent);`
	`191`	`+`
`177`	`192`	`if (!RelevantResults.IsPresent)`
`178`	`193`	`{`
`179`		`- WriteObject(finalResults, true);`
	`194`	`+ return finalResults;`
`180`	`195`	`}`
`181`	`196`	`else`
`182`	`197`	`{`
`183`	`198`	`var results = finalResults.FirstOrDefault(t => t.TableType == "RelevantResults")?`
`184`	`199`	`.ResultRows.Select(r => ConvertToPSObject(r));`
`185`		`- WriteObject(results, true);`
	`200`	`+ return results;`
`186`	`201`	`}`
`187`	`202`	`}`
`188`	`203`
	`204`	`+ protected override void ExecuteCmdlet()`
	`205`	`+ {`
	`206`	`+ var results = Run();`
	`207`	`+ WriteObject(results, true);`
	`208`	`+ }`
	`209`	`+`
`189`	`210`	`private object ConvertToPSObject(IDictionary<string, object> r)`
`190`	`211`	`{`
`191`	`212`	`PSObject res = new PSObject();`
`@@ -199,7 +220,7 @@ private object ConvertToPSObject(IDictionary<string, object> r)`
`199`	`220`	`return res;`
`200`	`221`	`}`
`201`	`222`
`202`		`- private KeywordQuery CreateKeywordQuery()`
	`223`	`+ private KeywordQuery CreateKeywordQuery(string clientFunction)`
`203`	`224`	`{`
`204`	`225`	`var keywordQuery = new KeywordQuery(ClientContext);`
`205`	`226`
`@@ -212,7 +233,7 @@ private KeywordQuery CreateKeywordQuery()`
`212`	`233`
`213`	`234`	`keywordQuery.QueryText = query;`
`214`	`235`	`keywordQuery.ClientType = ClientType;`
`215`		`- if (ParameterSpecified(nameof(TrimDuplicates))) keywordQuery.TrimDuplicates = TrimDuplicates;`
	`236`	`+ keywordQuery.TrimDuplicates = TrimDuplicates;`
`216`	`237`	`if (ParameterSpecified(nameof(Refiners))) keywordQuery.Refiners = Refiners;`
`217`	`238`	`if (ParameterSpecified(nameof(Culture))) keywordQuery.Culture = Culture;`
`218`	`239`	`if (ParameterSpecified(nameof(QueryTemplate))) keywordQuery.QueryTemplate = QueryTemplate;`
`@@ -245,7 +266,7 @@ private KeywordQuery CreateKeywordQuery()`
`245`	`266`	`{`
`246`	`267`	`SelectProperties = SelectProperties[0].Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);`
`247`	`268`	`}`
`248`		`-`
	`269`	`+`
`249`	`270`	`foreach (string property in SelectProperties)`
`250`	`271`	`{`
`251`	`272`	`selectProperties.Add(property.Trim());`
`@@ -289,6 +310,12 @@ private KeywordQuery CreateKeywordQuery()`
`289`	`310`	`keywordQuery.Properties.SetQueryPropertyValue(key, propVal);`
`290`	`311`	`}`
`291`	`312`	`}`
	`313`	`+`
	`314`	`+ QueryPropertyValue clientFuncPropVal = new QueryPropertyValue();`
	`315`	`+ clientFuncPropVal.StrVal = clientFunction;`
	`316`	`+ clientFuncPropVal.QueryPropertyValueTypeIndex = 1;`
	`317`	`+`
	`318`	`+ keywordQuery.Properties.SetQueryPropertyValue("ClientFunction", clientFuncPropVal);`
`292`	`319`	`return keywordQuery;`
`293`	`320`	`}`
`294`	`321`	`}`