Lost access to all agent nodes

[otavio]

The agents cannot access the control-planes directly but can using the load-balancer. However, control-planes access each other fine. Seems like a routing issue. Ideas?

Errors

Jan 02 07:03:14 infra-agent-fjn k3s[3529]: time="2024-01-02T07:03:14Z" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: failed to get CA certs: Get \"https://127.0.0.1:6444/cacerts\": read tcp 127.0.0.1:54128->127.0.0.1:6444: read: connection reset by peer (Client.Timeout exceeded while awaiting headers)"
Jan 02 07:03:37 infra-agent-fjn k3s[3529]: W0102 07:03:37.045551    3529 reflector.go:533] k8s.io/[email protected]/tools/cache/reflector.go:231: failed to list *v1.Endpoints: Get "https://127.0.0.1:6444/api/v1/namespaces/default/endpoints?fieldSelector=metadata.name%3Dkubernetes&limit=500&resourceVersion=0": net/http: TLS handshake timeout
Jan 02 07:03:37 infra-agent-fjn k3s[3529]: I0102 07:03:37.045621    3529 trace.go:236] Trace[30784480]: "Reflector ListAndWatch" name:k8s.io/[email protected]/tools/cache/reflector.go:231 (02-Jan-2024 07:03:27.044) (total time: 10001ms):
Jan 02 07:03:37 infra-agent-fjn k3s[3529]: Trace[30784480]: ---"Objects listed" error:Get "https://127.0.0.1:6444/api/v1/namespaces/default/endpoints?fieldSelector=metadata.name%3Dkubernetes&limit=500&resourceVersion=0": net/http: TLS handshake timeout 10001ms (07:03:37.045)
Jan 02 07:03:37 infra-agent-fjn k3s[3529]: Trace[30784480]: [10.001401683s] [10.001401683s] END
Jan 02 07:03:37 infra-agent-fjn k3s[3529]: E0102 07:03:37.045637    3529 reflector.go:148] k8s.io/[email protected]/tools/cache/reflector.go:231: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://127.0.0.1:6444/api/v1/namespaces/default/endpoints?fieldSelector=metadata.name%3Dkubernetes&limit=500&resourceVersion=0": net/http: TLS handshake timeout
Jan 02 07:03:39 infra-agent-fjn k3s[3529]: time="2024-01-02T07:03:39Z" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: failed to get CA certs: Get \"https://127.0.0.1:6444/cacerts\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"
Jan 02 07:04:04 infra-agent-fjn k3s[3529]: time="2024-01-02T07:04:04Z" level=info msg="Waiting to retrieve kube-proxy configuration; server is not ready: failed to get CA certs: Get \"https://127.0.0.1:6444/cacerts\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"

kube.tf

module "infra" {
  providers = {
    hcloud = hcloud
  }

  hcloud_token               = data.sops_file.secrets.data["hetzner-cloud-auth-token"]
  ssh_public_key             = file("../../secrets/keys/users/otavio.pub")
  ssh_additional_public_keys = [file("../../secrets/keys/users/victor.pub")]
  ssh_private_key            = null
  ssh_port                   = 60022

  source  = "kube-hetzner/kube-hetzner/hcloud"
  version = "2.11.3"

  # Extra k3s registries.
  k3s_registries = <<-EOT
    mirrors:
      docker.io:
        endpoint:
          - "https://mirror.gcr.io"
          - "https://registry-1.docker.io"
  EOT

  # Avoid updating it as Longhorn need check for compatibility.
  initial_k3s_channel = "v1.27"

  network_region       = "us-east"
  create_kubeconfig    = false
  create_kustomization = false

  # Use Cilium as network layer.
  cni_plugin = "cilium"

  # Block ICMP PING in nodes
  #block_icmp_ping_in = true

  dns_servers = [
    "1.1.1.1",
    "8.8.8.8",
    "2606:4700:4700::1111",
  ]

  extra_firewall_rules = [
    {
      direction       = "out"
      protocol        = "tcp"
      port            = "22"
      source_ips      = []
      destination_ips = ["0.0.0.0/0", "::/0"]
      description     = "Allow FluxCD access to resources via SSH"
    },
    {
      direction       = "out"
      protocol        = "tcp"
      port            = "587"
      source_ips      = []
      destination_ips = ["0.0.0.0/0", "::/0"]
      description     = "Allow SMTP access so we can send e-mails from pods"
    },
    {
      direction       = "in"
      protocol        = "udp"
      port            = "9993"
      source_ips      = ["0.0.0.0/0", "::/0"]
      destination_ips = []
      description     = "Allow ZeroTier access"
    },
    {
      direction       = "out"
      protocol        = "udp"
      port            = "9993"
      source_ips      = []
      destination_ips = ["0.0.0.0/0", "::/0"]
      description     = "Allow ZeroTier access"
    }
  ]

  use_control_plane_lb = true
  control_plane_nodepools = [
    {
      name        = "control-plane",
      server_type = "cpx31",
      location    = "ash",
      labels      = ["role=infra"],
      taints      = [],
      count       = 5
    }
  ]

  agent_nodepools = [
    {
      name        = "agent",
      server_type = "cpx41",
      location    = "ash",
      labels      = ["role=apps", "node.longhorn.io/create-default-disk=true"],
      taints      = [],
      count       = 9
    },
  ]

  kured_options = {
    "reboot-days" : "su"
    "start-time" : "9am"
    "end-time" : "5pm"
  }

  load_balancer_type     = "lb11"
  load_balancer_location = "ash"

  # We deploy the ingress ourselves.
  ingress_controller = "none"

  enable_longhorn     = true
  longhorn_namespace  = "storage"
  longhorn_values     = <<EOT
defaultSettings:
  createDefaultDiskLabeledNodes: true
  backupTargetCredentialSecret: longhorn-cloud-credentials
  backupTarget: s3://lab-infra-backup@us-central-1/longhorn/
  priorityClass: system-node-critical
  storageOverProvisioningPercentage: 100
  replicaSoftAntiAffinity: false
  allowVolumeCreationWithDegradedAvailability: false
  nodeDrainPolicy: allow-if-replica-is-stopped
persistence:
  reclaimPolicy: Retain
  recurringJobSelector:
    enable: true
    jobList: '[{"name":"infrequent-backups", "isGroup":true}]'
EOT
  disable_hetzner_csi = true

  cluster_name = "infra"

  use_cluster_name_in_node_name = true
}

[otavio]

We ended fixing this issue, we setted:

  # Use Cilium as network layer.
  cni_plugin = "cilium"
  cilium_version = "v1.14.5"
  cilium_routing_mode = "native"
  cilium_egress_gateway_enabled = false
  cilium_ipv4_native_routing_cidr = "10.0.0.0/8"