Skip to content

redhat 8.5 with later kernel build the drbd module in drbd-adapter job failed #87

New issue
New issue
Open
Open
redhat 8.5 with later kernel build the drbd module in drbd-adapter job failed#87
@san3Xian

Description

@san3Xian
san3Xian
opened on Sep 11, 2024

redhat 8.5 with later kernel cant not build the drbd module in drbd-adapter job

kubectl get deployments.apps -owide -n hwameistor hwameistor-operator
NAME                  READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS            IMAGES                                                         SELECTOR
hwameistor-operator   1/1     1            1           36d   hwameistor-operator   10.101.210.43/ghcr.m.daocloud.io/hwameistor/operator:v0.14.9   app=hwameistor-operator

log of shipper container:

kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c shipper
'/files/Dockerfile.shipper' -> '/pkgs/Dockerfile.shipper'
'/files/drbd-debs' -> '/pkgs/drbd-debs'
'/files/drbd-reactor.toml' -> '/pkgs/drbd-reactor.toml'
'/files/drbd-rpms' -> '/pkgs/drbd-rpms'
'/files/drbd.conf' -> '/pkgs/drbd.conf'
'/files/drbd.modules' -> '/pkgs/drbd.modules'
'/files/drbd.modules-load.conf' -> '/pkgs/drbd.modules-load.conf'
'/files/entrypoint.adapter.sh' -> '/pkgs/entrypoint.adapter.sh'
'/files/global_common.conf' -> '/pkgs/global_common.conf'
'/files/install_opam.sh' -> '/pkgs/install_opam.sh'
'/files/kernel-devels.centos7' -> '/pkgs/kernel-devels.centos7'
'/files/kernel-devels.centos8' -> '/pkgs/kernel-devels.centos8'
'/files/rhel7.0' -> '/pkgs/rhel7.0'
'/files/rhel7.1' -> '/pkgs/rhel7.1'
'/files/rhel7.2' -> '/pkgs/rhel7.2'
'/files/rhel7.3' -> '/pkgs/rhel7.3'
'/files/rhel7.4' -> '/pkgs/rhel7.4'
'/files/rhel7.5' -> '/pkgs/rhel7.5'
'/files/rhel7.6' -> '/pkgs/rhel7.6'
'/files/rhel7.7' -> '/pkgs/rhel7.7'
'/files/rhel7.8' -> '/pkgs/rhel7.8'
'/files/rhel7.9' -> '/pkgs/rhel7.9'
'/files/rhel8.0' -> '/pkgs/rhel8.0'
'/files/rhel8.1' -> '/pkgs/rhel8.1'
'/files/rhel8.2' -> '/pkgs/rhel8.2'
'/files/rhel8.3' -> '/pkgs/rhel8.3'
'/files/rhel8.4' -> '/pkgs/rhel8.4'
'/files/rhel8.5' -> '/pkgs/rhel8.5'
'/files/rhel8.6' -> '/pkgs/rhel8.6'
'/files/utils' -> '/pkgs/utils'

log of rhel8 container:

[root@mitqadaocappn02 ~]# kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c rhel8
+ which lbdisttool.py
/usr/local/bin/lbdisttool.py
++ lbdisttool.py -l
++ awk -F. '{print $1}'
+ image_dist=rhel8
++ lbdisttool.py -l --os-release /etc/host-release
++ awk -F. '{print $1}'
+ host_dist=rhel8
+ '[' -z rhel8 ']'
+ '[' -z rhel8 ']'
+ [[ rhel8 != rhel8 ]]
++ cat /proc/drbd
++ awk '/^version:/ {print $2}'
cat: /proc/drbd: No such file or directory
+ RUNNING_DRBD_VERSION=
+ '[' -z ']'
+ echo 'No DRBD Module is loaded'
+ echo OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64
+ export DRBD_EXIST=no
+ DRBD_EXIST=no
No DRBD Module is loaded
OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64
+ export DRBD_RMP_INSTALL=no
+ DRBD_RMP_INSTALL=no
+ export OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64
+ OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64
+ export OS_HOST_DIST=rhel8
+ OS_HOST_DIST=rhel8
+ [[ rhel8 == \k\y\l\i\n\1\0 ]]
+ LB_HOW=shipped_modules
+ [[ shipped_modules == \s\h\i\p\p\e\d\_\m\o\d\u\l\e\s ]]
+ bash -x /entry.sh
+ SIGN_KEY=https://packages.linbit.com/package-signing-pubkey.asc
+ PKGS=/pkgs
+ HOSTRELEASE=/etc/host-release
+ RPMDIR=/pkgs/drbd-rpms/
+ DEBDIR=/pkgs/drbd-debs/
+ HOW_DEPSONLY=deps_only
+ HOW_REPOFILE=repo_file
+ HOW_HASH=node_hash
+ HOW_FROMSRC=compile
+ HOW_FROMSHIPPED=shipped_modules
+ HOW_LOAD_FROM_RAM=RAM
+ HOW_INSTALL=install
+ modprobe_deps
+ local s
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe libcrc32c
+ s=success
+ debug 'Loading libcrc32c: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe nvmet_rdma
+ s=success
+ debug 'Loading nvmet_rdma: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe nvme_rdma
+ s=success
+ debug 'Loading nvme_rdma: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe loop
+ s=success
+ debug 'Loading loop: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_writecache
+ s=success
+ debug 'Loading dm_writecache: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_cache
+ s=success
+ debug 'Loading dm_cache: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_thin_pool
+ s=success
+ debug 'Loading dm_thin_pool: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_snapshot
+ s=success
+ debug 'Loading dm_snapshot: success'
+ '[' -n '' ']'
+ return 0
+ for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt
+ modprobe dm_crypt
+ s=success
+ debug 'Loading dm_crypt: success'
+ '[' -n '' ']'
+ return 0
+ return 0
+ [[ '' == \d\e\p\s\_\o\n\l\y ]]
+ grep -q '^drbd ' /proc/modules
+ '[' -z 4.18.0-477.10.1.el8_8.x86_64 ']'
+ var1=4.18.0-477.10.1.el8_8.x86_64
+ matchvar=4.18.0-477.10.1.el8_8.x86_64.rpm
++ find /pkgs/drbd-rpms/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.rpm' -print -quit
+ drbd_rpm=
+ '[' -n '' ']'
+ matchvar=4.18.0-477.10.1.el8_8.x86_64.deb
++ find /pkgs/drbd-debs/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.deb' -print -quit
+ drbd_deb=
+ [[ -n '' ]]
+ debug 'There is no corresponding kernel version deb package'
+ '[' -n '' ']'
+ return 0
+ pkgdir=/tmp/pkg
+ kodir=/tmp/ko
+ rm -rf /tmp/pkg /tmp/ko
+ mkdir -p /tmp/pkg /tmp/ko
+ fmt=rpm
++ type -p dpkg
+ '[' -n '' ']'
++ type -p emerge
+ '[' -n '' ']'
+ case $fmt in
++ repo::rpm::getrepofile
++ echo /etc/yum.repos.d/linbit.repo
+ repo=/etc/yum.repos.d/linbit.repo
++ how_to_get /etc/yum.repos.d/linbit.repo
++ local repo=/etc/yum.repos.d/linbit.repo
++ local how=
++ '[' -n '' ']'
++ '[' -f /etc/yum.repos.d/linbit.repo ']'
++ '[' -n '' ']'
++ mountpoint -q /usr/src
++ how=compile
++ echo compile
++ return 0
+ how_get=compile
+ debug 'Detected kmod method: "compile"'
+ '[' -n '' ']'
+ return 0
+ dist=we_do_not_care
++ needs_dist compile
++ local how=compile
++ local needsdist=n
++ [[ compile == \n\o\d\e\_\h\a\s\h ]]
++ echo n
++ return 0
+ need_dist=n
+ debug 'Needs host distribution info: "n"'
+ '[' -n '' ']'
+ return 0
+ [[ n == y ]]
+ case $how_get in
+ kos::fromsrc /tmp/pkg /tmp/ko
+ local pkgdir=/tmp/pkg
+ local kodir=/tmp/ko
+ cd /tmp/pkg
+ tar xf /drbd.tar.gz
+ cd drbd-9.0.32-1
+ make
Need a git checkout to regenerate drbd/.drbd_git_revision
make[1]: Entering directory '/tmp/pkg/drbd-9.0.32-1/drbd'

    Calling toplevel makefile of kernel source tree, which I believe is in
    KDIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/build

make -C /lib/modules/4.18.0-477.10.1.el8_8.x86_64/build   M=/tmp/pkg/drbd-9.0.32-1/drbd  modules
Makefile:1006: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel".  Stop.
make[1]: Leaving directory '/tmp/pkg/drbd-9.0.32-1/drbd'
make[1]: *** [Makefile:132: kbuild] Error 2
make: *** [Makefile:131: module] Error 2
+ [[ rhel8 == \k\y\l\i\n\1\0 ]]
++ how_to_load
++ [[ '' == yes ]]
++ echo RAM
++ return 0
+ how_load=RAM
+ debug 'Detected load method: "RAM"'
+ '[' -n '' ']'
+ return 0
+ [[ compile == \c\o\m\p\i\l\e ]]
+ [[ RAM == \i\n\s\t\a\l\l ]]
+ load_from_ram /tmp/pkg /tmp/ko
+ local pkgdir=/tmp/pkg
+ local kodir=/tmp/ko
+ find /tmp/pkg -name '*.ko' -exec cp '{}' /tmp/ko ';'
+ cd /tmp/ko
+ '[' '!' -f drbd.ko ']'
+ die 'Could not find the expexted *.ko, see stderr for more details'
+ echo

+ echo -e 'Could not find the expexted *.ko, see stderr for more details'
Could not find the expexted *.ko, see stderr for more details
+ exit 1
Successfully loaded shipped module
+ echo 'Successfully loaded shipped module'
DRBD IS ALREADY EXIST (yes or no):no
DRBD INSTALL BY PRM (yes or no) :no
+ echo 'DRBD IS ALREADY EXIST (yes or no):no'
+ echo 'DRBD INSTALL BY PRM (yes or no) :no'
+ [[ no == \y\e\s ]]
+ [[ yes == yes ]]
+ [[ rhel8 =~ rhel|kylin10 ]]
++ uname -r
+ KODIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd
+ mkdir -vp /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd
mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra'
mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd'
+ cp -vf '/tmp/ko/*.ko' /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd/
cp: cannot stat '/tmp/ko/*.ko': No such file or directory
+ depmod -a
+ cp -vf /pkgs/drbd.modules-load.conf /etc/modules-load.d/drbd.conf
'/pkgs/drbd.modules-load.conf' -> '/etc/modules-load.d/drbd.conf'
+ cp -vf /pkgs/drbd.modules /etc/sysconfig/modules/
'/pkgs/drbd.modules' -> '/etc/sysconfig/modules/drbd.modules'
+ cp -vf /pkgs/utils/drbdadm /pkgs/utils/drbdadm-83 /pkgs/utils/drbdadm-84 /pkgs/utils/drbdmeta /pkgs/utils/drbdsetup /pkgs/utils/drbdsetup-83 /pkgs/utils/drbdsetup-84 /usr-local-bin
'/pkgs/utils/drbdadm' -> '/usr-local-bin/drbdadm'
'/pkgs/utils/drbdadm-83' -> '/usr-local-bin/drbdadm-83'
'/pkgs/utils/drbdadm-84' -> '/usr-local-bin/drbdadm-84'
'/pkgs/utils/drbdmeta' -> '/usr-local-bin/drbdmeta'
'/pkgs/utils/drbdsetup' -> '/usr-local-bin/drbdsetup'
'/pkgs/utils/drbdsetup-83' -> '/usr-local-bin/drbdsetup-83'
'/pkgs/utils/drbdsetup-84' -> '/usr-local-bin/drbdsetup-84'
+ cat /pkgs/drbd.conf
+ cp -vf /pkgs/global_common.conf /etc/drbd.d/
'/pkgs/global_common.conf' -> '/etc/drbd.d/global_common.conf'
++ cat /proc/drbd
++ awk '/^version/ {print $2}'
cat: /proc/drbd: No such file or directory
+ [[ '' != 9.0.32-1 ]]
+ echo 'ERROR: DRBD is NOT loaded with the right version'
+ exit 1
ERROR: DRBD is NOT loaded with the right version
[root@hostname ~]#

pod yaml:

[root@mitqadaocappn02 ~]# kubectl get pods -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -oyaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: "2024-09-10T10:13:06Z"
  generateName: drbd-adapter-hostname-rhel8-1725963185-
  labels:
    app: drbd-adapter
    batch.kubernetes.io/controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9
    batch.kubernetes.io/job-name: drbd-adapter-hostname-rhel8-1725963185
    controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9
    drbd-version: v9.0.32-1
    job-name: drbd-adapter-hostname-rhel8-1725963185
  name: drbd-adapter-hostname-rhel8-1725963185-mn2n2
  namespace: hwameistor
  ownerReferences:
  - apiVersion: batch/v1
    blockOwnerDeletion: true
    controller: true
    kind: Job
    name: drbd-adapter-hostname-rhel8-1725963185
    uid: ebe37506-8320-4c60-afff-cd838643c1d9
  resourceVersion: "64000555"
  uid: 4e91e3e9-f5ef-4fef-aa3a-690cd2d21648
spec:
  containers:
  - image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2
    imagePullPolicy: IfNotPresent
    name: shipper
    resources: {}
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /pkgs
      name: pkgs
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-xvc27
      readOnly: true
  - command:
    - /pkgs/entrypoint.adapter.sh
    - 4.18.0-477.10.1.el8_8.x86_64
    env:
    - name: LB_SKIP
      value: "no"
    - name: LB_DROP
      value: "yes"
    - name: LB_UPGRADE
      value: "no"
    - name: LB_CHECK_HOSTNAME
      value: "yes"
    - name: NODE_NAME
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: spec.nodeName
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2
    imagePullPolicy: IfNotPresent
    name: rhel8
    resources: {}
    securityContext:
      privileged: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /pkgs
      name: pkgs
    - mountPath: /pkgs_root
      name: pkgroot
    - mountPath: /etc/host-release
      name: os-release
      readOnly: true
    - mountPath: /usr/src
      name: usr-src
      readOnly: true
    - mountPath: /lib/modules
      name: lib-modules
    - mountPath: /usr-local-bin
      name: usr-local-bin
    - mountPath: /etc/drbd.conf
      name: etc-drbd-conf
    - mountPath: /etc/drbd.d
      name: etc-drbd-d
    - mountPath: /var/lib/drbd
      name: var-lib-drbd
      readOnly: true
    - mountPath: /etc/modules-load.d
      name: etc-modules-load
    - mountPath: /etc/sysconfig/modules
      name: etc-sysconfig-modules
    - mountPath: /etc/centos-release
      name: centos-release
      readOnly: true
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-xvc27
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  hostNetwork: true
  hostPID: true
  nodeName: hostname
  nodeSelector:
    kubernetes.io/hostname: hostname
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Never
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 0
  tolerations:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
    operator: Exists
  - effect: NoSchedule
    key: node-role.kubernetes.io/control-plane
    operator: Exists
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - emptyDir: {}
    name: pkgs
  - hostPath:
      path: /root
      type: ""
    name: pkgroot
  - hostPath:
      path: /etc/os-release
      type: FileOrCreate
    name: os-release
  - hostPath:
      path: /etc/centos-release
      type: FileOrCreate
    name: centos-release
  - hostPath:
      path: /usr/src
      type: ""
    name: usr-src
  - hostPath:
      path: /lib/modules
      type: ""
    name: lib-modules
  - hostPath:
      path: /usr/local/bin
      type: ""
    name: usr-local-bin
  - hostPath:
      path: /etc/drbd.conf
      type: FileOrCreate
    name: etc-drbd-conf
  - hostPath:
      path: /etc/drbd.d
      type: DirectoryOrCreate
    name: etc-drbd-d
  - hostPath:
      path: /var/lib/drbd
      type: DirectoryOrCreate
    name: var-lib-drbd
  - hostPath:
      path: /etc/modules-load.d
      type: DirectoryOrCreate
    name: etc-modules-load
  - hostPath:
      path: /etc/sysconfig/modules
      type: DirectoryOrCreate
    name: etc-sysconfig-modules
  - name: kube-api-access-xvc27
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:19:37Z"
    status: "False"
    type: PodReadyToStartContainers
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    reason: PodFailed
    status: "False"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    reason: PodFailed
    status: "False"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2024-09-10T10:13:06Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2
    imageID: sha256:4ed5663ed4e00011ace7a6b11c846c4e45d3f1fe81720fe1b94f7f06fcf68ef9
    lastState: {}
    name: rhel8
    ready: false
    restartCount: 0
    started: false
    state:
      terminated:
        containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0
        exitCode: 1
        finishedAt: "2024-09-10T10:19:36Z"
        reason: Error
        startedAt: "2024-09-10T10:19:31Z"
  - containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f
    image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2
    imageID: sha256:d48131e7df28b10c410cacb77667f22c5bde6995015bc04e1a3a8c362f210388
    lastState: {}
    name: shipper
    ready: false
    restartCount: 0
    started: false
    state:
      terminated:
        containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f
        exitCode: 0
        finishedAt: "2024-09-10T10:13:07Z"
        reason: Completed
        startedAt: "2024-09-10T10:13:06Z"
  hostIP: 10.101.210.44
  hostIPs:
  - ip: 10.101.210.44
  phase: Failed
  podIP: 10.101.210.44
  podIPs:
  - ip: 10.101.210.44
  qosClass: BestEffort
  startTime: "2024-09-10T10:13:06Z"
[root@hostname ~]#

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions