Merge pull request #6 from hetzneronline/master

Merge Upstream into our fork

Author: guettli

almalinux.sh

@@ -92,6 +92,12 @@ generate_config_grub() {
 
   sed -i "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=\"$grub_cmdline_linux\"/" "$FOLD/hdd/etc/default/grub"
 
+  if ((IMG_VERSION > 85)) && ((IMG_VERSION <= 92)) && [[ -z "$GRUB_DEFAULT_OVERRIDE" ]]; then
+    GRUB_DEFAULT_OVERRIDE='saved'
+  fi
+
+  if [[ -n "$GRUB_DEFAULT_OVERRIDE" ]]; then sed -i "s/^GRUB_DEFAULT=.*/GRUB_DEFAULT=$GRUB_DEFAULT_OVERRIDE/" "$FOLD/hdd/etc/default/grub"; fi
+
   rm -f "$FOLD/hdd/boot/grub2/grub.cfg"
   if [ "$UEFI" -eq 1 ]; then
     execute_chroot_command "grub2-mkconfig -o /boot/efi/EFI/almalinux/grub.cfg 2>&1"; declare -i EXITCODE="$?"

archlinux.sh

@@ -33,7 +33,7 @@ extract_image() {
   local hdd_dir="$FOLD/hdd"
   debug '# extract archlinux-bootstrap to disk'
   debug "# run tar xzf $ARCHLINUX_BOOTSTRAP -C $hdd_dir"
-  tar xzf "$ARCHLINUX_BOOTSTRAP" -C "$hdd_dir" |& debugoutput || return 1
+  tar --zstd -xf "$ARCHLINUX_BOOTSTRAP" -C "$hdd_dir" |& debugoutput || return 1
 
   # pacman CheckSpace requires a mount to verify free space
   local chroot_dir="$hdd_dir/root.x86_64"

centos.sh

@@ -92,6 +92,12 @@ generate_config_grub() {
 
   sed -i "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=\"$grub_cmdline_linux\"/" "$FOLD/hdd/etc/default/grub"
 
+  if ((IMG_VERSION > 80)) && ((IMG_VERSION <= 90)) && ((IMG_VERSION != 610)) && [[ -z "$GRUB_DEFAULT_OVERRIDE" ]]; then
+    GRUB_DEFAULT_OVERRIDE='saved'
+  fi
+
+  if [[ -n "$GRUB_DEFAULT_OVERRIDE" ]]; then sed -i "s/^GRUB_DEFAULT=.*/GRUB_DEFAULT=$GRUB_DEFAULT_OVERRIDE/" "$FOLD/hdd/etc/default/grub"; fi
+
   rm -f "$FOLD/hdd/boot/grub2/grub.cfg"
   if [ "$UEFI" -eq 1 ]; then
     execute_chroot_command "grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg 2>&1"; declare -i EXITCODE="$?"

chroot.functions.sh

@@ -15,7 +15,7 @@ execute_chroot_command_wo_debug() {
   TMPDIR= unshare -f -p chroot "$FOLD/hdd" /usr/bin/env bash -c "$@"
   local r=$?
   for ((i=${#dirs[@]}-1; i>=0; i--)); do
-    until umount "$FOLD/hdd/${dirs[i]}"; do :; done
+    until umount -fl "$FOLD/hdd/${dirs[i]}"; do :; done
   done
   return $r
 }

config.sh

@@ -41,6 +41,7 @@ export SYSTEMD_NSPAWN_TMP_DIR="$FOLD/systemd_nspawn"
 export MODULES="virtio_pci virtio_blk via82cxxx sata_via sata_sil sata_nv sd_mod ahci atiixp raid0 raid1 raid5 raid6 raid10 3w-xxxx 3w-9xxx aacraid powernow-k8"
 export STATSSERVER4="88.198.31.148"
 export STATSSERVER6="[2a01:4f8:0:a0a1::bad:1]"
+export INSTALLATION_REPORT_URL='https://dcs-staging.hetzner.company/apis/data_center_metrics/v1/installimage_installation_reports'
 export HDDMINSIZE="7000000"
 
 export DNSRESOLVER=("185.12.64.1" "185.12.64.2")
@@ -95,7 +96,7 @@ export CPANEL_INSTALLER_SRC=http://mirror.hetzner.com/tools/cpanelinc/cpanel
 
 export PLESK_INSTALLER_SRC=http://mirror.hetzner.com/tools/parallels/plesk
 export PLESK_MIRROR=http://mirror.hetzner.com/plesk
-export PLESK_STD_VERSION=PLESK_18_0_57
+export PLESK_STD_VERSION=PLESK_18_0_62
 export PLESK_DOWNLOAD_RETRY_COUNT=999
 export PLESK_COMPONENTS=(
   awstats
@@ -138,13 +139,19 @@ export NOCOL="\033[00m"
 
 export ARCHLINUX_DIR="$SCRIPTPATH/../archlinux"
 export ARCHLINUX_RELEASE_KEY="$ARCHLINUX_DIR/release-key.pgp"
-export ARCHLINUX_BOOTSTRAP="$ARCHLINUX_DIR/archlinux-bootstrap-latest-x86_64.tar.gz"
+export ARCHLINUX_BOOTSTRAP="$ARCHLINUX_DIR/archlinux-bootstrap-latest-x86_64.tar.zst"
+
+export IMAGE_STILL_BETA_OVERRIDES=()
 
 other_images() {
   find "$IMAGESPATH"/ -maxdepth 1 -type f -name "CoreOS*" -a -not -name "*.sig" -printf '%f\n'
   echo 'Proxmox-Virtualization-Environment-on-Debian-Bookworm'
   echo 'Proxmox-Virtualization-Environment-on-Debian-Bullseye'
   find "$IMAGESPATH/" -maxdepth 1 -type f -iname '*beta*' -a -not -name '*.sig' -printf '%f\n'
+  local beta_image
+  for beta_image in "${IMAGE_STILL_BETA_OVERRIDES[@]}"; do
+    find "$IMAGESPATH/" -maxdepth 1 -type f -name "$beta_image" -a -not -name '*.sig' -printf '%f\n'
+  done
 }
 
 other_image_warning() {

functions.sh

@@ -153,6 +153,20 @@ generate_menu() {
     ;;
   esac
 
+  # remove still beta images unless menu is Other
+  if [[ "$1" != 'Other' ]]; then
+    local image beta_image new_rawlist=()
+    while read image; do
+      for beta_image in "${IMAGE_STILL_BETA_OVERRIDES[@]}"; do
+        if [[ "$image" == "$beta_image" ]]; then
+          continue 2
+        fi
+      done
+      new_rawlist+=("$image")
+    done <<< "$RAWLIST"
+    RAWLIST="$(printf '%s\n' "${new_rawlist[@]}")"
+  fi
+
   # generate formatted list for usage with "dialog"
   for i in $RAWLIST; do
     TEMPVAR="$i"
@@ -550,7 +564,7 @@ create_config() {
     local LIMIT=2096128
     local THREE_TB=2861588
     local DRIVE_SIZE; DRIVE_SIZE="$(sfdisk -s "$(smallest_hd)" 2>/dev/null)"
-    DRIVE_SIZE="$(echo "$DRIVE_SIZE" / 1024 | bc)"
+    DRIVE_SIZE="$(echo "$DRIVE_SIZE" / 1024 | bc)" # MiB
 
     # adjust swap dynamically according to RAM
     # RAM < 2 GB : SWAP=2 * RAM
@@ -569,6 +583,12 @@ create_config() {
       SWAPSIZE=$((RAM / 2 / 1024 + 1))
     fi
 
+    # revert swap size to 4G if swap wouldnt fit into smallest disk
+    DRIVE_SIZE_GIB=$((DRIVE_SIZE / 1024))
+    if (( SWAPSIZE >= DRIVE_SIZE_GIB )); then
+      SWAPSIZE=4
+    fi
+
     ESPPART='PART /boot/efi esp 256M\n'
 
     if [ "$UEFI" -eq 1 ]; then
@@ -1018,7 +1038,7 @@ validate_vars() {
     graph_error "ERROR: CentOS 6 is EOL since Nov 2020 and installimage does not support CentOS 6 anymore"
     return 1
   fi
-  if (( UEFI == 1 )) && rhel_9_based_image; then
+  if (( UEFI == 1 )) && rhel_9_based_image && test -z "$RHEL9_UEFI_OVERRIDE" ; then
     graph_error "ERROR: we do not yet support $IAM $IMG_VERSION on EFI systems"
     return 1
   fi
@@ -2200,8 +2220,11 @@ make_fstab_entry() {
   if grep -q '^/dev/disk/by-' <<< "$1"; then
     p='-part'
   else
-    p="$(echo $1 | grep nvme)"
-    [ -n "$p" ] && p='p'
+    local p; p=""
+    local nvme; nvme="$(echo $1 | grep nvme)"
+    [ -n "$nvme" ] && p='p'
+    local disk_by; disk_by="$(echo $1 | grep '^/dev/disk/by-')"
+    [ -n "$disk_by" ] && p='-part'
   fi
 
   if [ "$4" = "swap" ] ; then
@@ -2323,10 +2346,10 @@ make_swraid() {
         continue
       elif [ -n "$(echo "$line" | grep "/boot")" -a  "$metadata_boot" == "--metadata=0.90" ] || [ "$metadata" == "--metadata=0.90" ]; then
         # update fstab - replace /dev/sdaX with /dev/mdY
-        echo $line | sed "s/$SEDHDD\(p\)\?[0-9]\+/\/dev\/md$md_count/g" >> $fstab
+        echo $line | sed "s/$SEDHDD\(p\|-part\)\?[0-9]\+/\/dev\/md$md_count/g" >> $fstab
       else
         # update fstab - replace /dev/sdaX with /dev/md/Y
-        echo $line | sed "s/$SEDHDD\(p\)\?[0-9]\+/\/dev\/md\/$md_count/g" >> $fstab
+        echo $line | sed "s/$SEDHDD\(p\|-part\)\?[0-9]\+/\/dev\/md\/$md_count/g" >> $fstab
       fi
 
       # create raid array
@@ -2337,8 +2360,12 @@ make_swraid() {
         local n=0
         for n in $(seq 1 $COUNT_DRIVES) ; do
           TARGETDISK="$(eval echo \$DRIVE${n})"
-          local p="$(echo $TARGETDISK | grep nvme)"
-          [ -n "$p" ] && p='p'
+          local p; p=""
+          local nvme; nvme="$(echo $TARGETDISK | grep nvme)"
+          [ -n "$nvme" ] && p='p'
+          local disk_by; disk_by="$(echo $TARGETDISK | grep '^/dev/disk/by-')"
+          [ -n "$disk_by" ] && p='-part'
+
           components="$components $TARGETDISK$p$PARTNUM"
         done
 
@@ -2390,8 +2417,11 @@ make_lvm() {
   if [ -n "$1" ] ; then
     local fstab=$1
     local disk=$DRIVE1
-    local p; p="$(echo "$disk" | grep nvme)"
-    [ -n "$p" ] && p='p'
+    local p; p=""
+    local nvme; nvme="$(echo $TARGETDISK | grep nvme)"
+    [ -n "$nvme" ] && p='p'
+    local disk_by; disk_by="$(echo $TARGETDISK | grep '^/dev/disk/by-')"
+    [ -n "$disk_by" ] && p='-part'
 
     # TODO: needs to be removed
     # get device names for PVs depending if we use swraid or not
@@ -2421,7 +2451,7 @@ make_lvm() {
     # remove all Physical Volumes
     debug "# Removing all Physical Volumes"
     while read -r pv vg; do
-      if [[ "$vg" =~ $PRESERVE_VG ]]; then
+      if [[ -n "$PRESERVE_VG" ]] && [[ "$vg" =~ $PRESERVE_VG ]]; then
         debug "Not removing VG $vg"
       else
         pvremove -ff "$pv" 2>&1 | debugoutput
@@ -3850,7 +3880,8 @@ exit_function() {
   echo "  https://robot.hetzner.com/"
   echo
 
-  report_install
+  report_install_old
+  report_install_new
 }
 
 #function to check if it is a intel or amd cpu
@@ -3981,8 +4012,13 @@ function hdinfo() {
     *)
       local smartctl_json model_name serial_number
       if ! smartctl_json="$(smartctl -i -j "$dev" 2> /dev/null)"; then
-        echo '# unknown'
-        return
+        if is_usb_disk "$dev"; then
+          echo '# USB disk'
+          return
+        else
+          echo '# unknown'
+          return
+        fi
       fi
       model_name="$(jq -r '.model_name // empty' <<< "$smartctl_json" 2> /dev/null)" || :
       serial_number="$(disk_serial "$dev")" || :

install.sh

@@ -558,7 +558,8 @@ fi
 #
 # Report install.conf and debug.txt to $STATSSERVER
 #
-report_install
+report_install_old
+report_install_new
 
 #
 # Save installimage configuration and debug file on the new system

network_config.functions.sh

@@ -835,6 +835,8 @@ setup_network_config() {
          cat "$stderr_file" | debugoutput
          return 1
        fi
+       # delete chroot related lines from $stderr_file
+       sed -i '/Running in chroot/d' "$stderr_file"
        if [[ -s "$stderr_file" ]]; then
          debug 'fatal: netplan generate stderr not empty:'
          cat "$stderr_file" | debugoutput

passwd.functions.sh

@@ -32,7 +32,9 @@ rescue_gen_password_hash() {
     return 1
   fi
   cp /etc/shado{w,w.installimage_bak}
-  "$SCRIPTPATH/util/passwd$rescue_algo.sh" <<< "$password"$'\n'"$password" &> /dev/null || return 1
+  local passwd_cmd='passwd'
+  if [[ -e /usr/bin/passwd_real ]]; then passwd_cmd='passwd_real'; fi
+  PASSWD_CMD="$passwd_cmd" "$SCRIPTPATH/util/passwd$rescue_algo.sh" <<< "$password"$'\n'"$password" &> /dev/null || return 1
   local password_hash
   if ! password_hash="$(rescue_root_password_hash)"; then
     debug 'internal error: get rescue root password hash failed'