48248 dev (#21)

* Version 2.0 - added sync from KF to DC. (#12)
* Fixed issue with no input for either field type leading to a crash. Two other bugs fixed.
* Fixed issue with additional_emails field not syncing.
* Added independent logging via NLog.
* Added a switch to control syncing of fields deactivated in DigiCert.
* Added: automatic banned character recognition, paginated processing - as fixes for two bugs
* Added information on the new replacechar.json file.
* Fixes ab#47259
* Fixes ab#47725
* Fixes ab#46882

Author: mkachk

CHANGELOG.md

@@ -1,3 +1,8 @@
+Version 2.1.0
+
+    Added a system that gathers all non-Keyfactor friendly characters and allows the user to configure an alternative.
+    Added pagination based batch processing, memory consumption has been drastically reduced.
+
 Version 2.0.3  
 
     Added a setting to enable or disable syncing deactivated custom fields from DigiCert.

README.md

@@ -50,11 +50,17 @@ This should include the Keyfactor API endpoint, of the format https://domain.com
 This should include the common prefix all DigiCert certs have in your Keyfactor instance. For example, "DigiCert"
 - <b>ImportAllCustomDigicertFields</b>  
 This setting enables the tool to import all of the custom metadata fields included in DigiCert and sync all of their data.
-- <b>ReplaceDigicertWhiteSpaceCharacterInName</b>  
-In case the ImportAllCustomDigicertFields setting is used, this is necessary to for metadata field label conversion. DigiCert supports spaces in labels and Keyfactor does not, so this replaces the spaces in the name with your character sequence of choice.
+
+During the first run, the tool will scan the custom fields it will be importing for characters that are not supported in Keyfactor Metadata field names.
+Each unsupported character will be shown in a file named "replacechar.json" and its replacement can be selected. If the values in the file are not populated, the tool will not run a second time.
 - <b>ImportDataForDeactivatedDigiCertFields</b>  
 If this is enabled, custom metadata fields that were deactivated in DigiCert will also be synced, and the data stored in these fields in certificates will be too.
 
+### replacechar.json settings
+This file is populated during the first run of the tool if the ImportAllCustomDigicertFields setting is toggled. 
+The only text that needs replacing is shown as "null", and can be filled with any alphanumeric string. The "_" and "-" characters are also supported.
+
+
 ### manualfields.json settings
 This file is used to specify which metadata fields should be synced up.
 

digicert-metadata-sync/BannedCharacters.cs

@@ -0,0 +1,68 @@
+// Copyright 2021 Keyfactor
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Text.RegularExpressions;
+using Newtonsoft.Json.Linq;
+
+namespace DigicertMetadataSync;
+
+internal partial class DigicertSync
+{
+    public static List<CharDBItem> BannedCharacterParse(string input)
+    {
+        string pattern = "[a-zA-Z0-9-_]";
+
+        List<CharDBItem> bannedChars = new List<CharDBItem>();
+
+        foreach (char c in input)
+        {
+            if (!Regex.IsMatch(c.ToString(), pattern))
+            {
+                CharDBItem localitem = new CharDBItem();
+                localitem.character = c.ToString();
+                localitem.replacementcharacter = "null";
+                bannedChars.Add(localitem);
+            }
+        }
+
+        if (bannedChars.Count > 0)
+        {
+            Console.WriteLine("The field name " + input + " contains the following invalid characters: " +
+                              string.Join("", bannedChars.Select(item => item.character)));
+        }
+        else
+        {
+            Console.WriteLine("The field name " + input + " is valid.");
+        }
+
+        return bannedChars;
+    }
+
+    public static void CheckForChars(List<ReadInMetadataField> input, List<CharDBItem> allBannedChars, bool restartandconfigrequired)
+    {
+        foreach (var dgfield in input)
+        {
+            List<CharDBItem> newChars = BannedCharacterParse(dgfield.DigicertFieldName);
+            foreach (var newchar in newChars)
+            {
+                bool exists = allBannedChars.Any(allcharchar => allcharchar.character == newchar.character);
+                if (!exists)
+                {
+                    allBannedChars.Add(newchar);
+                    restartandconfigrequired = true;
+                }
+            }
+        }
+    }
+}

digicert-metadata-sync/Helpers.cs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+
+using System.Collections.Generic;
+
 using System.Text.RegularExpressions;
 using Newtonsoft.Json.Linq;
 
@@ -54,9 +57,13 @@ public static Dictionary<string, object> ClassConverter(object obj)
         return null;
     }
 
-    public static string ReplaceAllWhiteSpaces(string str, string replacement)
+    public static string ReplaceAllBannedCharacters(string input, List<CharDBItem>allBannedChars)
     {
-        return Regex.Replace(str, @"\s+", "_-_");
+        foreach (CharDBItem item in allBannedChars)
+        {
+            input = input.Replace(item.character, item.replacementcharacter);
+        }
+        return input;
     }
 
     public static bool CheckMode(string mode)
@@ -65,17 +72,18 @@ public static bool CheckMode(string mode)
         return false;
     }
 
-    private static List<KeyfactorMetadataInstanceSendoff> convertlisttokf(List<ReadInMetadataField> inputlist,
-        string replacementcharacter)
+    private static List<KeyfactorMetadataInstanceSendoff> convertlisttokf(List<ReadInMetadataField> inputlist, List<CharDBItem> allBannedChars, bool importallcustomfields)
     {
         var formattedlist = new List<KeyfactorMetadataInstanceSendoff>();
         if (inputlist.Count != 0)
             foreach (var input in inputlist)
             {
                 var formatinstance = new KeyfactorMetadataInstanceSendoff();
-                if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "")
-                    //If name is emtpy, use autocomplete.
-                    formatinstance.Name = ReplaceAllWhiteSpaces(input.DigicertFieldName, replacementcharacter);
+
+                if (input.KeyfactorMetadataFieldName == null || input.KeyfactorMetadataFieldName == "" || input.FieldType == "Custom")
+                    //If name is empty, clean up the characters.
+                    formatinstance.Name = ReplaceAllBannedCharacters(input.DigicertFieldName, allBannedChars);
+
                 else
                     //Use user input preferred name.
                     formatinstance.Name = input.KeyfactorMetadataFieldName;
@@ -86,6 +94,9 @@ private static List<KeyfactorMetadataInstanceSendoff> convertlisttokf(List<ReadI
                 formatinstance.Description = input.KeyfactorDescription;
                 formattedlist.Add(formatinstance);
             }
+
+
+
         return formattedlist;
     }