(CAT-2193): Fixed kubernetes environment setup for Debian. This allows spec tests to run successfully for debian platform

span786 · span786 · commit c009e7c40460 · 2025-01-08T17:27:51.000+05:30
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8', 'debian-11'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix: {'platform':['rhel-8'],'collection':['puppet7-nightly', 'puppet8-nightly']}
+      matrix: {'platform':['rhel-8', 'debian-11'],'collection':['puppet7-nightly', 'puppet8-nightly']}
 
     steps:
     - name: Checkout Source
diff --git a/manifests/repos.pp b/manifests/repos.pp
@@ -64,12 +64,13 @@
       'Debian': {
         $codename = fact('os.distro.codename')
         apt::source { 'kubernetes':
-          location => pick($kubernetes_apt_location, 'https://apt.kubernetes.io'),
-          repos    => pick($kubernetes_apt_repos, 'main'),
-          release  => pick($kubernetes_apt_release, 'kubernetes-xenial'),
+          location => pick($kubernetes_apt_location, 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/'),
+          repos    => pick($kubernetes_apt_repos, ' '),
+          release  => pick($kubernetes_apt_release, ' /'),
+          comment  => 'Kubernetes',
           key      => {
-            'id'     => pick($kubernetes_key_id, 'A362B822F6DEDC652817EA46B53DC80D13EDEF05'),
-            'source' => pick($kubernetes_key_source, 'https://packages.cloud.google.com/apt/doc/apt-key.gpg'),
+            'name'   => 'kubernetes-apt-keyring.gpg',
+            'source' => pick($kubernetes_key_source, 'https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key'),
           },
         }
 
diff --git a/spec/acceptance/kubernetes_spec.rb b/spec/acceptance/kubernetes_spec.rb
@@ -30,12 +30,12 @@ class {'kubernetes':
             }
           }
           /^(Debian|Ubuntu)$/: {
-          class {'kubernetes':
-                  controller => true,
-                  schedule_on_controller => true,
-                  environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
-                  ignore_preflight_errors => ['NumCPU'],
-                }
+            class {'kubernetes':
+              controller => true,
+              schedule_on_controller => true,
+              environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+              ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+            }
           }
           default:  {
             class {'kubernetes': } # any other OS are not supported
diff --git a/spec/spec_helper_acceptance_local.rb b/spec/spec_helper_acceptance_local.rb
@@ -40,12 +40,7 @@ def fetch_ip_hostname_by_role(role)
   platform = fetch_platform_by_node(ipaddr)
   ENV['TARGET_HOST'] = target_roles(role)[0][:name]
   hostname = run_shell('hostname').stdout.strip
-  os_family = run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip
-  int_ipaddr = if os_family.casecmp('redhat').zero?
-                 run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip
-               else
-                 run_shell("ip route get 8.8.8.8 | awk '{print $NF; exit}'").stdout.strip
-               end
+  int_ipaddr = run_shell("ip route get 8.8.8.8 | awk '{print $7; exit}'").stdout.strip
   [hostname, ipaddr, int_ipaddr]
 end
 
@@ -54,6 +49,10 @@ def change_target_host(role)
   ENV['TARGET_HOST'] = target_roles(role)[0][:name]
 end
 
+def os_family
+  run_shell("facter -y os.family | cut -d':' -f2 | tr -d ' '").stdout.strip
+end
+
 def reset_target_host
   ENV['TARGET_HOST'] = @orig_target_host
 end
@@ -70,37 +69,79 @@ def configure_puppet_server(controller, worker1, worker2)
   configure_puppet_agent('worker2')
   puppet_cert_sign
   # Create site.pp
-  site_pp = <<-EOS
-  node /#{controller[0]}/ {
-    class {'kubernetes':
-      kubernetes_version => '1.28.15',
-      kubernetes_package_version => '1.28.15',
-      controller_address => "#{controller[1]}:6443",
-      container_runtime => 'docker',
-      manage_docker => false,
-      controller => true,
-      schedule_on_controller => true,
-      environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
-      ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
-      cgroup_driver => 'systemd',
-      service_cidr => '10.138.0.0/12',
-    }
-  }
-  node /#{worker1}/ {
-    class {'kubernetes':
-      worker => true,
-      manage_docker => false,
-      cgroup_driver => 'systemd',
-    }
-  }
-  node /#{worker2}/  {
-    class {'kubernetes':
-      worker => true,
-      manage_docker => false,
-      cgroup_driver => 'systemd',
-    }
-  }
-  EOS
+
+  if os_family.casecmp('redhat').zero?
+    site_pp = <<-EOS
+      node /#{controller[0]}/ {
+        class {'kubernetes':
+          kubernetes_version => '1.28.15',
+          kubernetes_package_version => '1.28.15',
+          controller_address => "#{controller[1]}:6443",
+          container_runtime => 'docker',
+          manage_docker => false,
+          controller => true,
+          schedule_on_controller => true,
+          environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+          ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+          cgroup_driver => 'systemd',
+          service_cidr => '10.138.0.0/12',
+        }
+      }
+
+      node /#{worker1}/ {
+        class {'kubernetes':
+          worker => true,
+          manage_docker => false,
+          cgroup_driver => 'systemd',
+        }
+      }
+
+      node /#{worker2}/  {
+        class {'kubernetes':
+          worker => true,
+          manage_docker => false,
+          cgroup_driver => 'systemd',
+        }
+      }
+    EOS
+  else
+    site_pp = <<-EOS
+      node /#{controller[0]}/ {
+        class {'kubernetes':
+          kubernetes_version => '1.28.15',
+          kubernetes_package_version => '1.28.15-1.1',
+          controller_address => "#{controller[1]}:6443",
+          container_runtime => 'cri_containerd',
+          manage_docker => false,
+          controller => true,
+          schedule_on_controller => true,
+          environment  => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],
+          ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],
+          cgroup_driver => 'systemd',
+          service_cidr => '10.138.0.0/12',
+        }
+      }
+
+      node /#{worker1}/ {
+        class {'kubernetes':
+          kubernetes_package_version => '1.28.15-1.1',
+          worker => true,
+          manage_docker => false,
+          cgroup_driver => 'systemd',
+        }
+      }
+
+      node /#{worker2}/  {
+        class {'kubernetes':
+          kubernetes_package_version => '1.28.15-1.1',
+          worker => true,
+          manage_docker => false,
+          cgroup_driver => 'systemd',
+        }
+      }
+    EOS
+  end
+
   ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
   create_remote_file('site', '/etc/puppetlabs/code/environments/production/manifests/site.pp', site_pp)
   run_shell('chmod 644 /etc/puppetlabs/code/environments/production/manifests/site.pp')
@@ -133,8 +174,15 @@ def execute_agent(role)
 def reset_and_restart_containerd
   ['controller', 'worker1', 'worker2'].each do |node|
     ENV['TARGET_HOST'] = target_roles(node)[0][:name]
-    run_shell('rm -f /etc/containerd/config.toml')
-    run_shell('systemctl restart containerd')
+    if os_family.casecmp('redhat').zero?
+      run_shell('rm -f /etc/containerd/config.toml')
+      run_shell('systemctl restart containerd')
+    else
+      run_shell('wget https://github.com/containerd/containerd/releases/download/v1.6.12/containerd-1.6.12-linux-amd64.tar.gz && tar xvf containerd-1.6.12-linux-amd64.tar.gz')
+      run_shell('systemctl stop containerd')
+      run_shell('cd bin && cp * /usr/bin/')
+      run_shell('systemctl start containerd')
+    end
   end
 end
 
@@ -147,7 +195,11 @@ def open_communication_ports
       run_shell('iptables -I INPUT -p tcp -m multiport --dports 10251,10252,10255,30000:32767 -j ACCEPT')
     end
     run_shell('iptables -I INPUT -p udp -m multiport --dports 8472 -j ACCEPT')
-    run_shell('iptables-save > /etc/sysconfig/iptables')
+    if os_family.casecmp('redhat').zero?
+      run_shell('iptables-save > /etc/sysconfig/iptables')
+    else
+      run_shell('iptables-save > /etc/iptables/rules.v4')
+    end
   end
 end
 
@@ -157,6 +209,7 @@ def open_communication_ports
     hostname1, ipaddr1, int_ipaddr1 =  fetch_ip_hostname_by_role('controller')
     hostname2, ipaddr2, int_ipaddr2 =  fetch_ip_hostname_by_role('worker1')
     hostname3, ipaddr3, int_ipaddr3 =  fetch_ip_hostname_by_role('worker2')
+
     if c.filter.rules.key? :integration
       ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
       ['controller', 'worker1', 'worker2'].each do |node|
@@ -260,26 +313,48 @@ def open_communication_ports
     PUPPETCODE
 
     apply_manifest(pp)
-    if %r{debian|ubuntu-1604-lts}.match?(family)
+
+    if %r{debian|ubuntu}.match?(family)
       runtime = 'cri_containerd'
       cni = 'weave'
-      run_shell('apt-get update && apt-get install -y apt-transport-https')
-      run_shell('curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -')
-      run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list')
-      run_shell('apt-get update')
-      run_shell('apt-get install -y kubectl')
-      run_shell('sudo apt install docker-ce=18.06.0~ce~3-0~ubuntu  docker-ce-cli=18.06.0~ce~3-0~ubuntu -y')
-      run_shell('sudo apt install docker.io -y')
-      run_shell('systemctl start docker.service')
-      run_shell('systemctl enable docker.service')
-      if family.include?('ubuntu-1604-lts')
-        run_shell('sudo ufw disable')
-      else
-        # Workaround for debian as the strech repositories do not have updated kubernetes packages
-        run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" >> /etc/apt/sources.list.d/kube-xenial.list')
-        run_shell('/sbin/iptables -F')
+      ['controller', 'worker1', 'worker2'].each do |node|
+        ENV['TARGET_HOST'] = target_roles(node)[0][:name]
+
+        run_shell('apt update && apt install apt-transport-https -y')
+        run_shell('mkdir -p /etc/apt/keyrings')
+        run_shell('apt-get install -y curl gnupg2 software-properties-common')
+
+        if %r{debian-(10|11)}.match?(family)
+          run_shell('curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -')
+          run_shell('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"')
+        else
+          run_shell('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
+          run_shell('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"')
+        end
+        run_shell('curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg')
+        run_shell('echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list')
+
+        run_shell('apt-get update')
+        run_shell('apt install containerd -y')
+        run_shell('apt install kubectl kubelet kubeadm -y')
+        run_shell('apt-mark hold kubectl kubelet kubeadm')
+        run_shell('apt install docker-ce docker-ce-cli -y')
+        run_shell('apt install docker.io -y')
+        run_shell('systemctl start docker.service')
+        run_shell('systemctl enable docker.service')
+        run_shell('echo iptables-persistent iptables-persistent/autosave_v4 boolean true | sudo debconf-set-selections')
+        run_shell('echo iptables-persistent iptables-persistent/autosave_v6 boolean true | sudo debconf-set-selections')
+        run_shell('apt install iptables-persistent -y')
+        if family.include?('ubuntu')
+          run_shell('sudo ufw disable')
+        else
+          # Workaround for debian as the strech repositories do not have updated kubernetes packages
+          # run_shell('echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" >> /etc/apt/sources.list.d/kube-xenial.list')
+          run_shell('/sbin/iptables -F')
+        end
       end
     end
+
     if %r{rhel|redhat|centos}.match?(family)
       runtime = 'docker'
       cni = 'weave'
@@ -302,7 +377,8 @@ def open_communication_ports
     end
 
     ENV['TARGET_HOST'] = target_roles('controller')[0][:name]
-    run_shell('docker build -t kubetool:latest /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling')
+
+    run_shell('docker build -t kubetool:latest --network host /etc/puppetlabs/code/environments/production/modules/kubernetes/tooling')
 
     docker_run = <<~DOCKER
       docker run --rm -v $(pwd)/hieradata:/mnt -e OS=#{family} \
@@ -317,6 +393,7 @@ def open_communication_ports
     DOCKER
 
     run_shell(docker_run)
+
     create_remote_file('nginx', '/tmp/nginx.yml', nginx)
     create_remote_file('hiera', '/etc/puppetlabs/puppet/hiera.yaml', hiera)
     run_shell('chmod 644 /etc/puppetlabs/puppet/hiera.yaml')

Original file line number	Diff line number	Diff line change
`@@ -30,12 +30,12 @@ class {'kubernetes':`
`30`	`30`	`}`
`31`	`31`	`}`
`32`	`32`	`/^(Debian\|Ubuntu)$/: {`
`33`		`- class {'kubernetes':`
`34`		`- controller => true,`
`35`		`- schedule_on_controller => true,`
`36`		`- environment => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],`
`37`		`- ignore_preflight_errors => ['NumCPU'],`
`38`		`- }`
	`33`	`+ class {'kubernetes':`
	`34`	`+ controller => true,`
	`35`	`+ schedule_on_controller => true,`
	`36`	`+ environment => ['HOME=/root', 'KUBECONFIG=/etc/kubernetes/admin.conf'],`
	`37`	`+ ignore_preflight_errors => ['NumCPU','ExternalEtcdVersion'],`
	`38`	`+ }`
`39`	`39`	`}`
`40`	`40`	`default: {`
`41`	`41`	`class {'kubernetes': } # any other OS are not supported`