Allow KUBECONFIG environment variable to point to multiple files (#411)

jkotalik · web-flow · commit 324a3e72fd17 · 2020-04-22T15:17:45.000-07:00
* Allow KUBECONFIG environment variable to point to multiple files

* Add more tests, add API (can make internal if necessary)

* test

* allow passing in env var

* small amount of feedback

* Feedback

* Nits

* Some extra tests and comments
diff --git a/src/KubernetesClient/KubeConfigModels/K8SConfiguration.cs b/src/KubernetesClient/KubeConfigModels/K8SConfiguration.cs
@@ -9,6 +9,7 @@ namespace k8s.KubeConfigModels
     /// </summary>
     /// <remarks>
     /// Should be kept in sync with https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/tools/clientcmd/api/v1/types.go
+    /// Should update MergeKubeConfig in KubernetesClientConfiguration.ConfigFile.cs if updated.
     /// </remarks>
     public class K8SConfiguration
     {
diff --git a/src/KubernetesClient/KubernetesClientConfiguration.ConfigFile.cs b/src/KubernetesClient/KubernetesClientConfiguration.ConfigFile.cs
@@ -1,7 +1,7 @@
 using System;
+using System.Collections.Generic;
 #if NETSTANDARD2_0
 using Newtonsoft.Json;
-using System.Collections.Generic;
 using System.Diagnostics;
 #endif
 using System.IO;
@@ -30,34 +30,45 @@ public partial class KubernetesClientConfiguration
         /// </summary>
         public string CurrentContext { get; private set; }
 
+        // For testing
+        internal static string KubeConfigEnvironmentVariable { get; set; } = "KUBECONFIG";
+
         /// <summary>
         ///     Initializes a new instance of the <see cref="KubernetesClientConfiguration" /> from default locations
         ///     If the KUBECONFIG environment variable is set, then that will be used.
         ///     Next, it looks for a config file at <see cref="KubeConfigDefaultLocation"/>.
         ///     Then, it checks whether it is executing inside a cluster and will use <see cref="InClusterConfig()" />.
         ///     Finally, if nothing else exists, it creates a default config with localhost:8080 as host.
         /// </summary>
+        /// <remarks>
+        ///     If multiple kubeconfig files are specified in the KUBECONFIG environment variable,
+        ///     merges the files, where first occurence wins. See https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#merging-kubeconfig-files.
+        /// </remarks>
         public static KubernetesClientConfiguration BuildDefaultConfig()
         {
-            var kubeconfig = Environment.GetEnvironmentVariable("KUBECONFIG");
+            var kubeconfig = Environment.GetEnvironmentVariable(KubeConfigEnvironmentVariable);
             if (kubeconfig != null)
             {
-                return BuildConfigFromConfigFile(kubeconfigPath: kubeconfig);
+                var configList = kubeconfig.Split(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ';' : ':').Select((s) => new FileInfo(s));
+                var k8sConfig = LoadKubeConfig(configList.ToArray());
+                return BuildConfigFromConfigObject(k8sConfig);
             }
+
             if (File.Exists(KubeConfigDefaultLocation))
             {
                 return BuildConfigFromConfigFile(kubeconfigPath: KubeConfigDefaultLocation);
             }
+
             if (IsInCluster())
             {
                 return InClusterConfig();
             }
+
             var config = new KubernetesClientConfiguration();
             config.Host = "http://localhost:8080";
             return config;
         }
 
-
         /// <summary>
         ///     Initializes a new instance of the <see cref="KubernetesClientConfiguration" /> from config file
         /// </summary>
@@ -564,6 +575,46 @@ public static K8SConfiguration LoadKubeConfig(Stream kubeconfigStream)
             return LoadKubeConfigAsync(kubeconfigStream).GetAwaiter().GetResult();
         }
 
+        /// <summary>
+        ///     Loads Kube Config
+        /// </summary>
+        /// <param name="kubeconfigs">List of kube config file contents</param>
+        /// <param name="useRelativePaths">When <see langword="true"/>, the paths in the kubeconfig file will be considered to be relative to the directory in which the kubeconfig
+        /// file is located. When <see langword="false"/>, the paths will be considered to be relative to the current working directory.</param>
+        /// <returns>Instance of the <see cref="K8SConfiguration"/> class</returns>
+        /// <remarks>
+        ///     The kube config files will be merges into a single <see cref="K8SConfiguration"/>, where first occurence wins.
+        ///     See https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#merging-kubeconfig-files.
+        /// </remarks>
+        internal static K8SConfiguration LoadKubeConfig(FileInfo[] kubeConfigs, bool useRelativePaths = true)
+        {
+            return LoadKubeConfigAsync(kubeConfigs, useRelativePaths).GetAwaiter().GetResult();
+        }
+
+        /// <summary>
+        ///     Loads Kube Config
+        /// </summary>
+        /// <param name="kubeconfigs">List of kube config file contents</param>
+        /// <param name="useRelativePaths">When <see langword="true"/>, the paths in the kubeconfig file will be considered to be relative to the directory in which the kubeconfig
+        /// file is located. When <see langword="false"/>, the paths will be considered to be relative to the current working directory.</param>
+        /// <returns>Instance of the <see cref="K8SConfiguration"/> class</returns>
+        /// <remarks>
+        ///     The kube config files will be merges into a single <see cref="K8SConfiguration"/>, where first occurence wins.
+        ///     See https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#merging-kubeconfig-files.
+        /// </remarks>
+        internal static async Task<K8SConfiguration> LoadKubeConfigAsync(FileInfo[] kubeConfigs, bool useRelativePaths = true)
+        {
+            var basek8SConfig = await LoadKubeConfigAsync(kubeConfigs[0], useRelativePaths).ConfigureAwait(false);
+
+            for (var i = 1; i < kubeConfigs.Length; i++)
+            {
+                var mergek8SConfig = await LoadKubeConfigAsync(kubeConfigs[i], useRelativePaths).ConfigureAwait(false);
+                MergeKubeConfig(basek8SConfig, mergek8SConfig);
+            }
+
+            return basek8SConfig;
+        }
+
         /// <summary>
         /// Tries to get the full path to a file referenced from the Kubernetes configuration.
         /// </summary>
@@ -593,5 +644,76 @@ private static string GetFullPath(K8SConfiguration configuration, string path)
                 return Path.Combine(Path.GetDirectoryName(configuration.FileName), path);
             }
         }
+
+        /// <summary>
+        /// Merges kube config files together, preferring configuration present in the base config over the merge config.
+        /// </summary>
+        /// <param name="basek8SConfig">The <see cref="K8SConfiguration"/> to merge into</param>
+        /// <param name="mergek8SConfig">The <see cref="K8SConfiguration"/> to merge from</param>
+        private static void MergeKubeConfig(K8SConfiguration basek8SConfig, K8SConfiguration mergek8SConfig)
+        {
+            // For scalar values, prefer local values 
+            basek8SConfig.CurrentContext = basek8SConfig.CurrentContext ?? mergek8SConfig.CurrentContext;
+            basek8SConfig.FileName = basek8SConfig.FileName ?? mergek8SConfig.FileName;
+
+            // Kinds must match in kube config, otherwise throw.
+            if (basek8SConfig.Kind != mergek8SConfig.Kind)
+            {
+                throw new KubeConfigException($"kubeconfig \"kind\" are different between {basek8SConfig.FileName} and {mergek8SConfig.FileName}");
+            }
+
+            if (mergek8SConfig.Preferences != null)
+            {
+                foreach (var preference in mergek8SConfig.Preferences)
+                {
+                    if (basek8SConfig.Preferences?.ContainsKey(preference.Key) == false)
+                    {
+                        basek8SConfig.Preferences[preference.Key] = preference.Value;
+                    }
+                }
+            }
+
+            if (mergek8SConfig.Extensions != null)
+            {
+                foreach (var extension in mergek8SConfig.Extensions)
+                {
+                    if (basek8SConfig.Extensions?.ContainsKey(extension.Key) == false)
+                    {
+                        basek8SConfig.Extensions[extension.Key] = extension.Value;
+                    }
+                }
+            }
+
+            // Note, Clusters, Contexts, and Extensions are map-like in config despite being represented as a list here:
+            // https://github.com/kubernetes/client-go/blob/ede92e0fe62deed512d9ceb8bf4186db9f3776ff/tools/clientcmd/api/types.go#L238
+            basek8SConfig.Clusters = MergeLists(basek8SConfig.Clusters, mergek8SConfig.Clusters, (s) => s.Name);
+            basek8SConfig.Users = MergeLists(basek8SConfig.Users, mergek8SConfig.Users, (s) => s.Name);
+            basek8SConfig.Contexts = MergeLists(basek8SConfig.Contexts, mergek8SConfig.Contexts, (s) => s.Name);
+        }
+
+        private static IEnumerable<T> MergeLists<T>(IEnumerable<T> baseList, IEnumerable<T> mergeList, Func<T, string> getNameFunc)
+        {
+            if (mergeList != null && mergeList.Count() > 0)
+            {
+                var mapping = new Dictionary<string, T>();
+                foreach (var item in baseList)
+                {
+                    mapping[getNameFunc(item)] = item;
+                }
+
+                foreach (var item in mergeList)
+                {
+                    var name = getNameFunc(item);
+                    if (!mapping.ContainsKey(name))
+                    {
+                        mapping[name] = item;
+                    }
+                }
+
+                return mapping.Values;
+            }
+
+            return baseList;
+        }
     }
 }
diff --git a/tests/KubernetesClient.Tests/KubernetesClientConfigurationTests.cs b/tests/KubernetesClient.Tests/KubernetesClientConfigurationTests.cs
@@ -1,5 +1,7 @@
+using System;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Threading;
 using k8s.Exceptions;
 using k8s.KubeConfigModels;
@@ -407,6 +409,102 @@ public void LoadKubeConfigStream()
             AssertConfigEqual(expectedCfg, cfg);
         }
 
+        [Fact]
+        public void LoadKubeConfigFromEnvironmentVariable()
+        {
+            // BuildDefaultConfig assumes UseRelativePaths: true, which isn't
+            // done by any tests.
+            var filePath = Path.GetFullPath("assets/kubeconfig.relative.yml");
+            var environmentVariable = "KUBECONFIG_LoadKubeConfigFromEnvironmentVariable";
+
+            Environment.SetEnvironmentVariable(environmentVariable, filePath);
+            KubernetesClientConfiguration.KubeConfigEnvironmentVariable = environmentVariable;
+
+            var cfg = KubernetesClientConfiguration.BuildDefaultConfig();
+
+            Assert.NotNull(cfg);
+        }
+
+        [Fact]
+        public void LoadKubeConfigFromEnvironmentVariable_MultipleConfigs()
+        {
+            // This test makes sure that a list of environment variables works (no exceptions),
+            // doesn't check validity of configuration, which is done in other tests.
+
+            var filePath = Path.GetFullPath("assets/kubeconfig.relative.yml");
+            var environmentVariable = "KUBECONFIG_LoadKubeConfigFromEnvironmentVariable_MultipleConfigs";
+
+            Environment.SetEnvironmentVariable(environmentVariable, string.Concat(filePath, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ';' : ':', filePath));
+            KubernetesClientConfiguration.KubeConfigEnvironmentVariable = environmentVariable;
+
+            var cfg = KubernetesClientConfiguration.BuildDefaultConfig();
+
+            Assert.NotNull(cfg);
+        }
+
+        [Fact]
+        public void LoadSameKubeConfigFromEnvironmentVariableUnmodified()
+        {
+            var txt = File.ReadAllText("assets/kubeconfig.yml");
+            var expectedCfg = Yaml.LoadFromString<K8SConfiguration>(txt);
+
+            var fileInfo = new FileInfo(Path.GetFullPath("assets/kubeconfig.yml"));
+
+            var cfg = KubernetesClientConfiguration.LoadKubeConfig(new FileInfo[] { fileInfo, fileInfo });
+
+            AssertConfigEqual(expectedCfg, cfg);
+        }
+
+        [Fact]
+        public void MergeKubeConfigNoDuplicates()
+        {
+            var firstPath = Path.GetFullPath("assets/kubeconfig.as-user-extra.yml");
+            var secondPath = Path.GetFullPath("assets/kubeconfig.yml");
+
+            var cfg = KubernetesClientConfiguration.LoadKubeConfig(new FileInfo[] { new FileInfo(firstPath), new FileInfo(secondPath) });
+
+            // Merged file has 6 users now.
+            Assert.Equal(6, cfg.Users.Count());
+            Assert.Equal(5, cfg.Clusters.Count());
+            Assert.Equal(5, cfg.Contexts.Count());
+        }
+
+        [Fact]
+        public void AlwaysPicksFirstOccurence()
+        {
+            var firstPath = Path.GetFullPath("assets/kubeconfig.no-cluster.yml");
+            var secondPath = Path.GetFullPath("assets/kubeconfig.no-context.yml");
+
+            var cfg = KubernetesClientConfiguration.LoadKubeConfig(new FileInfo[] { new FileInfo(firstPath), new FileInfo(secondPath) });
+
+            var user = cfg.Users.Where(u => u.Name == "green-user").Single();
+            Assert.NotNull(user.UserCredentials.Password);
+            Assert.Null(user.UserCredentials.ClientCertificate);
+        }
+
+        [Fact]
+        public void ContextFromSecondWorks()
+        {
+            var firstPath = Path.GetFullPath("assets/kubeconfig.no-current-context.yml");
+            var secondPath = Path.GetFullPath("assets/kubeconfig.no-user.yml");
+
+            var cfg = KubernetesClientConfiguration.LoadKubeConfig(new FileInfo[] { new FileInfo(firstPath), new FileInfo(secondPath) });
+
+            // green-user
+            Assert.NotNull(cfg.CurrentContext);
+        }
+
+        [Fact]
+        public void ContextPreferencesExtensionsMergeWithDuplicates()
+        {
+            var path = Path.GetFullPath("assets/kubeconfig.preferences-extensions.yml");
+
+            var cfg = KubernetesClientConfiguration.LoadKubeConfig(new FileInfo[] { new FileInfo(path), new FileInfo(path) });
+
+            Assert.Equal(1, cfg.Extensions.Count);
+            Assert.Equal(1, cfg.Preferences.Count);
+        }
+
         /// <summary>
         ///     Ensures Kube config file can be loaded from within a non-default <see cref="SynchronizationContext"/>.
         ///     The use of <see cref="UIFactAttribute"/> ensures the test is run from within a UI-like <see cref="SynchronizationContext"/>.
diff --git a/tests/KubernetesClient.Tests/assets/kubeconfig.preferences-extensions.yml b/tests/KubernetesClient.Tests/assets/kubeconfig.preferences-extensions.yml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Config
+preferences:
+  colors: true
+extensions:
+  foo: bar
diff --git a/tests/KubernetesClient.Tests/assets/kubeconfig.user-pass.yml b/tests/KubernetesClient.Tests/assets/kubeconfig.user-pass.yml
@@ -19,4 +19,4 @@ users:
 - name: green-user
   user:
     password: secret
-    username: admin
+    username: admin

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@ namespace k8s.KubeConfigModels`
`9`	`9`	`/// </summary>`
`10`	`10`	`/// <remarks>`
`11`	`11`	`/// Should be kept in sync with https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/tools/clientcmd/api/v1/types.go`
	`12`	`+ /// Should update MergeKubeConfig in KubernetesClientConfiguration.ConfigFile.cs if updated.`
`12`	`13`	`/// </remarks>`
`13`	`14`	`public class K8SConfiguration`
`14`	`15`	`{`