Use deb822 sources format everywhere

Author: ArrayBolt3

chroot-script

@@ -92,6 +92,139 @@ markauto() {
 }
 # }}}
 
+# write a deb822 sources stanza to a file {{{
+writesource() {
+  local file types uris suites components signed_by
+
+  file="${1:-}"
+  types="${2:-}"
+  uris="${3:-}"
+  suites="${4:-}"
+  components="${5:-}"
+  signed_by="${6:-}"
+
+  mkdir -p "$(dirname "${file}")"
+
+  if [ -f "$file" ]; then
+    ## stanzas need a space between them and the previous stanza
+    echo '' >> "$file"
+  fi
+
+  echo "Types: $types" >> "$file"
+  echo "URIs: $uris" >> "$file"
+  echo "Suites: $suites" >> "$file"
+  echo "Components: $components" >> "$file"
+  if [ -n "$signed_by" ]; then
+    echo "Signed-By: $signed_by" >> "$file"
+  fi
+}
+# }}}
+
+# delete deb822 sources stanzas from a file if they match the specified
+# variables {{{
+deletesource() {
+  local file types uris suites components signed_by
+
+  file="${1:-}"
+  types="${2:-}"
+  uris="${3:-}"
+  suites="${4:-}"
+  components="${5:-}"
+  signed_by="${6:-}"
+
+  local stanza_contents current_types current_uris current_suites \
+    current_components current_signed_by line do_omit file_contents
+  stanza_contents=""
+  current_types=""
+  current_uris=""
+  current_suites=""
+  current_components=""
+  current_signed_by=""
+  do_omit='y'
+  file_contents=""
+
+  while read -r line; do
+    if [[ "$line" =~ ^[:space:]*# ]]; then
+      if [ -z "$stanza_contents" ]; then
+        file_contents+="$line"$'\n'
+      else
+        stanza_contents+="$line"$'\n'
+      fi
+      continue
+    fi
+
+    line="$(sed 's/^[[:space:]]*//; s/[[:space:]]*$//;' <<< "$line")"
+    if [ -z "$line" ]; then
+      if [ -n "$types" ] \
+        && [ "$types" != "$current_types" ]; then
+        do_omit='n'
+      elif [ -n "$uris" ] \
+        && [ "$uris" != "$current_uris" ]; then
+        do_omit='n'
+      elif [ -n "$suites" ] \
+        && [ "$suites" != "$current_suites" ]; then
+        do_omit='n'
+      elif [ -n "$components" ] \
+        && [ "$components" != "$current_components" ]; then
+        do_omit='n'
+      elif [ -n "$signed_by" ] \
+        && [ "$signed_by" != "$current_signed_by" ]; then
+        do_omit='n'
+      elif [ -z "$current_types" ] \
+        && [ -z "$current_uris" ] \
+        && [ -z "$current_suites" ] \
+        && [ -z "$current_components" ] \
+        && [ -z "$current_signed_by" ]; then
+        do_omit='n'
+      fi
+
+      if [ "$do_omit" = 'n' ]; then
+        file_contents+="$stanza_contents"$'\n'
+      fi
+
+      stanza_contents=""
+      current_types=""
+      current_uris=""
+      current_suites=""
+      current_components=""
+      current_signed_by=""
+      do_omit='y'
+
+      continue
+    fi
+
+    stanza_contents+="$line"$'\n'
+    if [[ "$line" =~ ^Types: ]]; then
+      current_types="$(cut -d' ' -f2- <<< "$line")"
+      continue
+    fi
+    if [[ "$line" =~ ^URIs: ]]; then
+      current_uris="$(cut -d' ' -f2- <<< "$line")"
+      continue
+    fi
+    if [[ "$line" =~ ^Suites: ]]; then
+      current_suites="$(cut -d' ' -f2- <<< "$line")"
+      continue
+    fi
+    if [[ "$line" =~ ^Components: ]]; then
+      current_components="$(cut -d' ' -f2- <<< "$line")"
+      continue
+    fi
+    if [[ "$line" =~ ^Signed-By: ]]; then
+      current_signed_by="$(cut -d' ' -f2- <<< "$line")"
+      continue
+    fi
+
+  # Extra echo piped in so that the variable parsing code runs for the last
+  # stanza
+  done < <(cat "$file"; echo '')
+
+  # Strip leading and trailing newliens
+  file_contents="$(sed -z 's/^\n*//; s/\n*$//;' <<< "$file_contents")"
+  echo "$file_contents" > "$file"
+}
+# }}}
+
 # define chroot mirror {{{
 chrootmirror() {
   if [ "$KEEP_SRC_LIST" = "yes" ] ; then
@@ -105,8 +238,11 @@ chrootmirror() {
   echo "Using repository components $COMPONENTS"
 
   if [ -n "$MIRROR" ] ; then
-    echo "Adjusting sources.list for mirror (${MIRROR})."
-    echo "deb $MIRROR $RELEASE $COMPONENTS" > /etc/apt/sources.list
+    echo "Deleting obsolete sources.list"
+    rm -f /etc/apt/sources.list
+
+    echo "Adjusting sources.list.d/debian.sources for mirror (${MIRROR})."
+    writesource '/etc/apt/sources.list.d/debian.sources' 'deb' "$MIRROR" "$RELEASE" "$COMPONENTS" '/usr/share/keyrings/debian-archive-keyring.gpg'
   fi
 
   # add security.debian.org:
@@ -115,8 +251,8 @@ chrootmirror() {
     *)
       # bullseye and newer releases use a different repository layout, see
       # https://lists.debian.org/debian-devel-announce/2019/07/msg00004.html
-      echo "Adding security.debian.org/debian-security to sources.list."
-      echo "deb http://security.debian.org/debian-security ${RELEASE}-security $COMPONENTS" >> /etc/apt/sources.list
+      echo "Adding security.debian.org/debian-security to sources.list.d/debian.sources."
+      writesource '/etc/apt/sources.list.d/debian.sources' 'deb' 'http://security.debian.org/debian-security' "${RELEASE}-security" "$COMPONENTS" '/usr/share/keyrings/debian-archive-keyring.gpg'
       ;;
   esac
 }
@@ -130,11 +266,10 @@ remove_chrootmirror() {
   fi
 
   if [ -n "$MIRROR" ] && echo "$MIRROR" | grep -q 'file:' ; then
-    echo "Removing local mirror (${MIRROR}) from sources.list."
-    TMP_MIRROR="${MIRROR//\//\\\/}"
-    sed -i "/deb $TMP_MIRROR $RELEASE $COMPONENTS/ D" /etc/apt/sources.list
-    echo "Adding fallback mirror entry (${FALLBACK_MIRROR}) to sources.list instead."
-    echo "deb $FALLBACK_MIRROR $RELEASE $COMPONENTS" >> /etc/apt/sources.list
+    echo "Removing local mirror (${MIRROR}) from sources.list.d/debian.sources."
+    deletesource '/etc/apt/sources.list.d/debian.sources' 'deb' "$MIRROR" "$RELEASE" "$COMPONENTS"
+    echo "Adding fallback mirror entry (${FALLBACK_MIRROR}) to sources.list.d/debian.sources instead."
+    writesource '/etc/apt/sources.list.d/debian.sources' 'deb' "$FALLBACK_MIRROR" "$RELEASE" "$COMPONENTS" '/usr/share/keyrings/debian-archive-keyring.gpg'
   fi
 }
 # }}}
@@ -146,19 +281,15 @@ grmlrepos() {
   fi
 
   # user might have provided their own apt sources configuration
-  if [ -r /etc/apt/sources.list.d/grml.list ] ; then
-    echo "File /etc/apt/sources.list.d/grml.list exists already, not modifying."
+  if [ -r /etc/apt/sources.list.d/grml.sources ] ; then
+    echo "File /etc/apt/sources.list.d/grml.sources exists already, not modifying."
   else
-    echo "Setting up /etc/apt/sources.list.d/grml.list."
-    cat > /etc/apt/sources.list.d/grml.list << EOF
-# grml: stable repository:
-  deb     [signed-by=/usr/share/keyrings/grml-archive-keyring.gpg] http://deb.grml.org/ grml-stable main
-  deb-src [signed-by=/usr/share/keyrings/grml-archive-keyring.gpg] http://deb.grml.org/ grml-stable main
-
-# grml: testing/development repository:
-  deb     [signed-by=/usr/share/keyrings/grml-archive-keyring.gpg] http://deb.grml.org/ grml-testing main
-  deb-src [signed-by=/usr/share/keyrings/grml-archive-keyring.gpg] http://deb.grml.org/ grml-testing main
-EOF
+    echo "Setting up /etc/apt/sources.list.d/grml.sources."
+    echo '# grml: stable repository:' >> /etc/apt/sources.list.d/grml.sources
+    writesource '/etc/apt/sources.list.d/grml.sources' 'deb deb-src' 'https://deb.grml.org' 'grml-stable' 'main' '/usr/share/keyrings/grml-archive-keyring.gpg'
+    echo '' >> /etc/apt/sources.list.d/grml.sources
+    echo '# grml: testing/development repository:' >> /etc/apt/sources.list.d/grml.sources
+    writesource '/etc/apt/sources.list.d/grml.sources' 'deb deb-src' 'https://deb.grml.org' 'grml-testing' 'main' '/usr/share/keyrings/grml-archive-keyring.gpg'
   fi
 
   # make sure we install packages from Grml's pool only if not available from Debian
@@ -188,11 +319,8 @@ EOF
 # feature to provide Debian backports repos {{{
 backportrepos() {
   if [ -n "$BACKPORTREPOS" ] ; then
-    cat >> /etc/apt/sources.list.d/backports.list << EOF
-# debian backports: ${RELEASE}-backports repository:
-deb     ${MIRROR} ${RELEASE}-backports main
-deb-src ${MIRROR} ${RELEASE}-backports main
-EOF
+    echo "# debian backports: ${RELEASE}-backports repository:" >> /etc/apt/sources.list.d/backports.sources
+    writesource '/etc/apt/sources.list.d/backports.sources' 'deb deb-src' "$MIRROR" "${RELEASE}-backports" 'main' '/usr/share/keyrings/debian-archive-keyring.gpg'
   fi
 }
 # }}}

config

@@ -41,17 +41,19 @@
 # BOOT_APPEND='pci=nomsi'
 
 # Set mirror where packages will be downloaded from.
-# Default: use /etc/debootstrap/etc/apt/sources.list if it exists, else
-# 'http://deb.debian.org/debian'
+# Default: use /etc/debootstrap/etc/apt/sources.list.d/debian.sources if it
+# exists, else use /etc/debootstrap/etc/apt/sources.list if it exists, else
+# use 'http://deb.debian.org/debian'
 # Usage example:
 # MIRROR='ftp://ftp.de.debian.org/debian'
 
-# If /etc/apt/sources.list should NOT be build on the fly, this
-# options allows providing a separate apt sources.list file via
-# /etc/debootstrap/etc/apt/sources.list
+# If /etc/apt/sources.list.d/debian.sources should NOT be built on the fly,
+# this option allows providing a separate apt sources.list file via
+# /etc/debootstrap/etc/apt/sources.list, or providing a deb822-format sources
+# file via /etc/debootstrap/etc/apt/sources.list.d/debian.sources.
 # KEEP_SRC_LIST='yes'
 
-# To enable the Grml repository via /etc/apt/sources.list.d/grml.list
+# To enable the Grml repository via /etc/apt/sources.list.d/grml.sources
 # set this variable to 'yes'.
 # Default: 'no'
 # GRMLREPOS='yes'
@@ -62,7 +64,7 @@
 # GRMLPACKAGES='grml-etc-core'
 
 # To enable Debian's backports repository via
-# /etc/apt/sources.list.d/backports.list set this variable to 'yes'.
+# /etc/apt/sources.list.d/backports.sources set this variable to 'yes'.
 # Default: 'no'
 # BACKPORTREPOS='yes'
 
@@ -71,7 +73,7 @@
 # Default: 'trixie'
 # RELEASE='trixie'
 
-# Define components that should be used within sources.list.
+# Define components that should be used within sources.list.d/debian.sources.
 # Default: 'main'
 # COMPONENTS='main contrib non-free'
 

grml-debootstrap

@@ -139,7 +139,7 @@ Configuration options:
       --debconf <file>     Pre-seed packages using specified pre-seed db file.
       --grmlrepos          Enable Grml's Debian repository (deb.grml.org).
       --backportrepos      Enable Debian's backports repository (backports.debian.org).
-      --keep_src_list      Do not overwrite user provided apt sources.list.
+      --keep_src_list      Do not override user provided apt sources.
       --contrib            Enable 'contrib' in COMPONENTS (defaults to 'main' only).
       --non-free           Enable non-free / non-free-firmware in COMPONENTS (defaults to 'main' only).
       --hostname <name>    Hostname of Debian system.
@@ -461,7 +461,7 @@ while :; do
     shift; _opt_chroot_scripts="$1"
     _opt_chroot_scripts_set=T
     ;;
-  --keep_src_list)     # Do not overwrite user provided apt sources.list
+  --keep_src_list)     # Do not overwrite user provided apt sources
     _opt_keep_src_list=T
     ;;
   --hostname)          # Hostname of Debian system

grml-debootstrap.8.txt

@@ -40,8 +40,8 @@ Options and environment variables
 *--backportrepos*::
 
     Enable Debian's Backports repository via the
-    /etc/apt/sources.list.d/backports.list configuration file. This provides the
-    option to install Debian packages from Debian's backports repository.
+    /etc/apt/sources.list.d/backports.sources configuration file. This provides
+    the option to install Debian packages from Debian's backports repository.
 
 *--bootappend* _appendline-for-the-kernel_::
 
@@ -113,7 +113,7 @@ Options and environment variables
 *--grmlrepos*::
 
     Enable Grml.org's Debian repository (http://deb.grml.org/) via the
-    /etc/apt/sources.list.d/grml.list configuration file. This provides the
+    /etc/apt/sources.list.d/grml.sources configuration file. This provides the
     option to install Debian packages from Grml's repository.
 
 *--grub* _device_::
@@ -135,7 +135,8 @@ Options and environment variables
 
 *--keep_src_list*::
 
-     Do not overwrite user provided /etc/apt/sources.list.
+     Do not override user provided /etc/apt/sources.list.d/debian.sources or
+     delete /etc/apt/sources.list.
 
 *-m*, *--mirror* _URL_::
 

packer/debian64_provision.sh

@@ -152,9 +152,12 @@ EOF
 sources_list_setup() {
   # This is ugly because it's 'testing' no matter what ISO we're using, but otherwise we're running into
   # W: Failed to fetch http://snapshot.debian.org/archive/debian/20141114/dists/testing/main/binary-amd64/Packages  404  Not Found [IP: 193.62.202.30 80]
-  echo "* Setting up /etc/apt/sources.list.d/debian.list to avoid snapshot.debian.org usage causing possible failures"
-  cat > /etc/apt/sources.list.d/debian.list << EOF
-deb http://ftp.debian.org/debian testing main
+  echo "* Setting up /etc/apt/sources.list.d/debian.sources to avoid snapshot.debian.org usage causing possible failures"
+  cat > /etc/apt/sources.list.d/debian.sources << EOF
+Types: deb
+URIs: http://ftp.debian.org/debian
+Suites: testing
+Components: main
 EOF
 }
 
@@ -163,8 +166,11 @@ grml_debootstrap_setup() {
   if [ "$GRML_DEBOOTSTRAP_VERSION" = "latest" ] ; then
     echo "** GRML_DEBOOTSTRAP_VERSION is set to '$GRML_DEBOOTSTRAP_VERSION'"
     echo "** Setting up grml-debootstrap from CI repository from jenkins.grml.org"
-    cat > /etc/apt/sources.list.d/grml-debootstrap.list << EOF
-deb     http://jenkins.grml.org/debian grml-debootstrap main
+    cat > /etc/apt/sources.list.d/grml-debootstrap.sources << EOF
+Types: deb
+URIs: http://jenkins.grml.org/debian
+Suites: grml-debootstrap
+Components: main
 EOF
     wget -O - http://jenkins.grml.org/debian/C525F56752D4A654.asc | apt-key add -
     apt-get update