Feature/stability (#226)

* cxone crawl stability

* fix issue #225

* scan stat record stabilization

* documentation updates

* enable odata api

Author: nleach999

Libs/Configuration/Impls/CxSASTAPIOverrides.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Configuration;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CxAnalytix.Configuration.Impls
+{
+    public sealed class CxSASTAPIOverrides : ConfigurationElement
+    {
+
+        [ConfigurationProperty("Project", IsRequired = false, DefaultValue = false)]
+        public bool Project
+        {
+            get => (bool)this["Project"];
+            set => this["Project"] = value;
+        }
+    }
+}

Libs/Configuration/Impls/CxSASTConnection.cs

@@ -14,5 +14,11 @@ public String MNOUrl
             set { this["mnoURL"] = value; }
         }
 
+        [ConfigurationProperty("UseOdata", IsRequired = false)]
+        public CxSASTAPIOverrides Overrides
+        {
+            get => (CxSASTAPIOverrides)this["UseOdata"];
+            set => this["UseOdata"] = value;
+        }
 	}
 }

Libs/CxRestClient/SAST/CxProjects.cs

@@ -18,7 +18,11 @@ public class CxProjects
     {
         private static ILog _log = LogManager.GetLogger(typeof(CxProjects));
 
-        private static String URL_SUFFIX = "cxrestapi/projects";
+        private static readonly String REST_URL_SUFFIX = "cxrestapi/projects";
+
+        private static readonly int ODATA_TOP = 25;
+        private static readonly String ODATA_URL_SUFFIX = "cxwebinterface/odata/v1/Projects?$expand=CustomFields&$select=OwningTeamId,PresetId,Id,Name,IsPublic" +
+            $"&$orderby=Id asc&$top={ODATA_TOP}";
 
         private static String _apiVersion = null;
 
@@ -41,29 +45,94 @@ private static String GetApiVersion(CxSASTRestContext ctx, CancellationToken tok
         private CxProjects()
         { }
 
+        #region DTOs
+
         [JsonObject(MemberSerialization.OptIn)]
         public class ProjectCustomFields
         {
             [JsonProperty(PropertyName = "name")]
             public String FieldName { get; internal set; }
+
+            [JsonProperty(PropertyName = "FieldName")]
+            private String odata_FieldName
+            {
+                get => FieldName;
+                set => FieldName = value;
+            }
+
+
             [JsonProperty(PropertyName = "value")]
             public String FieldValue { get; internal set; }
+
+            [JsonProperty(PropertyName = "FieldValue")]
+            private String odata_FieldValue
+            {
+                get => FieldValue;
+                set => FieldValue = value;
+            }
+
         }
 
         [JsonObject(MemberSerialization.OptIn)]
         public class Project
         {
             [JsonProperty(PropertyName = "teamId")]
             public String TeamId { get; internal set; }
+            [JsonProperty(PropertyName = "OwningTeamId")]
+            private String odata_TeamId
+            {
+                get => TeamId;
+                set => TeamId = value;
+            }
+
+
+
             public int PresetId { get; internal set; }
+
+
             [JsonProperty(PropertyName = "id")]
             public int ProjectId { get; internal set; }
+            [JsonProperty(PropertyName = "Id")]
+            private int odata_ProjectId
+            {
+                get => ProjectId;
+                set => ProjectId = value;
+            }
+
+
             [JsonProperty(PropertyName = "name")]
             public String ProjectName { get; internal set; }
+            [JsonProperty(PropertyName = "Name")]
+            private String odata_ProjectName
+            {
+                get => ProjectName;
+                set => ProjectName = value;
+            }
+
+
+
+
             [JsonProperty(PropertyName = "isPublic")]
             public bool IsPublic { get; internal set; }
+            [JsonProperty(PropertyName = "IsPublic")]
+            private bool odata_IsPublic
+            {
+                get => IsPublic;
+                set => IsPublic = value;
+            }
+
+
+
             [JsonProperty(PropertyName = "customFields")]
             public List<ProjectCustomFields> CustomFields { get; internal set; }
+            [JsonProperty(PropertyName = "CustomFields")]
+            private List<ProjectCustomFields> odata_CustomFields
+            {
+                get => CustomFields;
+                set => CustomFields = value;
+            }
+
+
 
             [JsonProperty(PropertyName = "isBranched")]
             public bool IsBranched { get; internal set; }
@@ -78,6 +147,7 @@ public class Project
             public override string ToString() =>
                 $"{ProjectId}:{ProjectName} [TeamId: {TeamId} Public: {IsPublic} CustomFields: {CustomFields.Count}]";
 		}
+        #endregion
 
         private class ProjectReader : IEnumerable<Project>, IEnumerator<Project>, IDisposable
         {
@@ -161,12 +231,57 @@ public void Reset()
             {
                 throw new NotImplementedException();
             }
+        }
 
+        private static IEnumerable<Project> GetProjects_odata(CxSASTRestContext ctx, CancellationToken token)
+        {
+            List<Project> returnedResults = new();
+
+            var filter = new Dictionary<String, String>();
+            List<Project> fetchedPage = null;
+
+            do
+            {
+                String requestUrl = UrlUtils.MakeUrl(ctx.Sast.ApiUrl, ODATA_URL_SUFFIX, filter);
 
+                using (var projectReader = WebOperation.ExecuteGet<ProjectReader>(
+                ctx.Sast.Json.CreateClient
+                , (response) =>
+                {
+                    using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result))
+                    using (var jtr = new JsonTextReader(sr))
+                    {
+                        JToken jt = JToken.Load(jtr);
+
+                        return new ProjectReader(jt["value"], ctx, token);
+                    }
+                }
+                , requestUrl
+                , ctx.Sast
+                , token))
+                    fetchedPage = new List<Project>(projectReader);
+
+                if (fetchedPage != null)
+                {
+                    returnedResults.AddRange(fetchedPage);
+                    filter["$filter"] = $"id gt {fetchedPage[fetchedPage.Count - 1].ProjectId}";
+                }
+
+
+            } while (fetchedPage != null && fetchedPage.Count == ODATA_TOP);
+
+            return returnedResults;
         }
 
+        public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, CancellationToken token, bool useOData)
+        {
+            if (useOData)
+                return GetProjects_odata(ctx, token);
+            else
+                return GetProjects_rest(ctx, token);
+        }
 
-        public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, CancellationToken token)
+        private static IEnumerable<Project> GetProjects_rest(CxSASTRestContext ctx, CancellationToken token)
         {
             using (var projectReader = WebOperation.ExecuteGet<ProjectReader>(
             ctx.Sast.Json.CreateClient
@@ -180,7 +295,7 @@ public static IEnumerable<Project> GetProjects(CxSASTRestContext ctx, Cancellati
                     return new ProjectReader(jt, ctx, token);
                 }
             }
-            , UrlUtils.MakeUrl(ctx.Sast.ApiUrl, URL_SUFFIX)
+            , UrlUtils.MakeUrl(ctx.Sast.ApiUrl, REST_URL_SUFFIX)
             , ctx.Sast
             , token, apiVersion: GetApiVersion(ctx, token) ))
                 return new List<Project>(projectReader);

Libs/CxRestClient/SAST/CxScanStatistics.cs

@@ -1,4 +1,5 @@
 using CxAnalytix.Exceptions;
+using CxAnalytix.Extensions;
 using CxRestClient.Utility;
 using log4net;
 using Newtonsoft.Json;
@@ -7,6 +8,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Linq.Expressions;
 using System.Net;
 using System.Net.Http;
 using System.Text;
@@ -270,24 +272,51 @@ internal FullScanStatistics()
 
         public static FullScanStatistics GetScanFullStatistics(CxSASTRestContext ctx, CancellationToken token, String scanId)
         {
-            var statistics = GetScanStatistics(ctx, token, scanId);
-
-            var pf = GetScanParsedFiles(ctx, token, scanId);
-            var fq = GetScanFailedQueries(ctx, token, scanId);
-            var fgq = GetScanFailedGeneralQueries(ctx, token, scanId);
-            var sgq = GetScanSuccessfulGeneralQueries(ctx, token, scanId);
-
-            if (statistics.Result == null)
-                return null;
-
-            return new FullScanStatistics()
-            {
-                Statistics = statistics.Result,
-                ParsedFiles = pf.Result,
-                FailedQueries = fq.Result,
-                FailedGeneralQueries = fgq.Result,
-                SuccessGeneralQueries = sgq.Result
-            };
+            CancellationTokenSource localToken = new();
+
+            List<Task> runningTasks = new();
+
+            using (token.Register(() => localToken.Cancel()))
+                try
+                {
+                    var statistics = GetScanStatistics(ctx, localToken.Token, scanId);
+                    runningTasks.Add(statistics);
+
+                    var pf = GetScanParsedFiles(ctx, localToken.Token, scanId);
+                    runningTasks.Add(pf);
+
+                    var fq = GetScanFailedQueries(ctx, localToken.Token, scanId);
+                    runningTasks.Add(fq);
+
+                    var fgq = GetScanFailedGeneralQueries(ctx, localToken.Token, scanId);
+                    runningTasks.Add(fgq);
+
+                    var sgq = GetScanSuccessfulGeneralQueries(ctx, localToken.Token, scanId);
+                    runningTasks.Add(sgq);
+
+
+                    if (statistics.Result != null)
+                        return new FullScanStatistics()
+                        {
+                            Statistics = statistics.Result,
+                            ParsedFiles = pf.Result,
+                            FailedQueries = fq.Result,
+                            FailedGeneralQueries = fgq.Result,
+                            SuccessGeneralQueries = sgq.Result
+                        };
+                }
+                catch (Exception)
+                {
+                    localToken.Cancel();
+                    throw;
+                }
+                finally
+                {
+                    runningTasks.SafeWaitAllToEnd();
+                    runningTasks.DisposeTasks();
+                }
+
+            return null;
         }
 
     }

Libs/Exceptions/ScanCrawlException.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CxAnalytix.Exceptions
+{
+    public class ScanCrawlException : Exception
+    {
+        private static String MakeMessage(String scanId, String projectName, String teamName) =>
+            $"Exception caught processing scan [{scanId}] in [{projectName}]" +
+                                        (!String.IsNullOrEmpty(teamName) ? $" assigned to team(s) [{teamName}]" : "");
+
+
+        public ScanCrawlException (String scanId, String projectName, String teamName) 
+            : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName) )
+        {
+
+        }
+        public ScanCrawlException(String scanId, String projectName, String teamName, Exception ex) 
+            : base(ScanCrawlException.MakeMessage(scanId, projectName, teamName), ex)
+        {
+
+        }
+    }
+}

Libs/Executive/ExecuteLoop.cs

@@ -11,7 +11,7 @@ namespace CxAnalytix.Executive
 {
     public class ExecuteLoop : ExecuteOnce
     {
-        private static readonly ILog appLog = LogManager.GetLogger(typeof(ExecuteLoop));
+        private static readonly ILog _log = LogManager.GetLogger(typeof(ExecuteLoop));
 
         public static new void Execute(CancellationTokenSource t)
         {
@@ -35,19 +35,23 @@ public class ExecuteLoop : ExecuteOnce
                 }
                 catch (Exception ex)
                 {
-                    appLog.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
+                    _log.Error("Vulnerability data transformation aborted due to unhandled exception.", ex);
                 }
 
 
                 GC.Collect();
 
-                Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token).Wait();
+                using (var delay = Task.Delay(Service.ProcessPeriodMinutes * 60 * 1000, t.Token))
+                    delay.Wait(t.Token);
+
             } while (!t.Token.IsCancellationRequested);
 
+            _log.Info("Execution complete, ending.");
+
         }
         private static void Fatal(Exception ex, CancellationTokenSource ct)
         {
-            appLog.Error("Fatal exception caught, program ending.", ex);
+            _log.Error("Fatal exception caught, program ending.", ex);
             ct.Cancel();
             Process.GetCurrentProcess().Kill(true);