From 0cc23fe813cd7ed5d813100a86ff824f79a0da44 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Tue, 3 Sep 2024 15:12:55 +1000 Subject: [PATCH 01/10] Linting and style updates --- bt-bugfix | 162 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 85 insertions(+), 77 deletions(-) diff --git a/bt-bugfix b/bt-bugfix index dfaba3f..5bf3bb6 100755 --- a/bt-bugfix +++ b/bt-bugfix @@ -8,13 +8,13 @@ # Free Software Foundation; either version 3 of the License, or (at your # option) any later version. -fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; } -warning() { echo "WARNING [$(basename $0)]: $@"; } -info() { echo "INFO [$(basename $0)]: $@"; } +fatal() { echo "FATAL [$(basename "$0")]: $*" 1>&2; exit 1; } +warning() { echo "WARNING [$(basename "$0")]: $*" 1>&2; } +info() { echo "INFO [$(basename "$0")]: $*"; } usage() { cat< $new_changelog < "$new_changelog" <> $new_changelog <> "$new_changelog" <> $new_changelog <> "$new_changelog" <> $new_changelog <> "$new_changelog" <> $new_changelog < $(date +"%a, %d %b %Y %H:%M:%S %z") EOF -cat $old_changelog >> $new_changelog +cat "$old_changelog" >> "$new_changelog" info "Updating turnkey_version and preparing update version package." -echo "$new_name" > $rootfs/etc/turnkey_version -$BT/bin/generate-release-deb $new_changelog $rootfs +echo "$new_name" > "$rootfs/etc/turnkey_version" +"$BT/bin/generate-release-deb" "$new_changelog" "$rootfs" update_patch=$BT/patches/update-release conf_script=$update_patch/conf -mkdir -p $update_patch -touch $conf_script -cat > $conf_script < "$conf_script" < $O/$new_name.manifest -$BT/bin/generate-buildenv iso $appname > $O/$new_name.iso.buildenv -if [ -e $BT_PROFILES/$appname ]; then - mkdir -p $O/$new_name.tklbam +"$BT/bin/generate-signature" "$O/$new_isofile" +"$BT/bin/generate-manifest" "$rootfs" > "$O/$new_name.manifest" +"$BT/bin/generate-buildenv" iso "$appname" > "$O/$new_name.iso.buildenv" +if [[ -e "$BT_PROFILES/$appname" ]]; then + mkdir -p "$O/$new_name.tklbam" export PROFILES_CONF=$BT_PROFILES - $BT/bin/generate-tklbam-profile $O/$new_name.iso $O/$new_name.tklbam + "$BT/bin/generate-tklbam-profile" "$O/$new_name.iso" "$O/$new_name.tklbam" fi _cleanup -if [ "$publish" == "yes" ]; then - $BT/bin/iso-publish $BT_ISOS/$new_name.iso - if [[ -z "BT_DEBUG" ]]; then - rm -rf $BT_ISOS/$name.iso* - rm -rf $BT_ISOS/$new_name* +if [[ "$publish" == "yes" ]]; then + "$BT/bin/iso-publish" "$BT_ISOS/$new_name.iso" + if [[ -z "$BT_DEBUG" ]]; then + rm -rf "$BT_ISOS/$name.iso*" + rm -rf "${BT_ISOS:?}/${new_name:?}*" fi fi From 2bf8e4809e36764e7a3135a0bece3a2c2927c141 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 5 Sep 2024 15:30:39 +1000 Subject: [PATCH 02/10] rename 'updates' patch -> apt-upgrade --- patches/{updates => apt-upgrade}/conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename patches/{updates => apt-upgrade}/conf (100%) diff --git a/patches/updates/conf b/patches/apt-upgrade/conf similarity index 100% rename from patches/updates/conf rename to patches/apt-upgrade/conf From c2b84b60521f0e66c469b4437fdfc63627c86005 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 5 Sep 2024 16:20:49 +1000 Subject: [PATCH 03/10] Spilt apt-upgrade patch and create new clean-old-kernels patch --- patches/apt-upgrade/conf | 15 +++++---------- patches/clean-old-kernels/conf | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) create mode 100755 patches/clean-old-kernels/conf diff --git a/patches/apt-upgrade/conf b/patches/apt-upgrade/conf index 68d8062..d4027ae 100755 --- a/patches/apt-upgrade/conf +++ b/patches/apt-upgrade/conf @@ -1,16 +1,11 @@ #!/bin/bash -ex -list_file=/var/lib/apt/lists/archive.turnkeylinux.org_debian_dists_bullseye_Release -[[ -f "$list_file" ]] || apt-get update +export DEBIAN_FRONTEND=noninteractive -DEBIAN_FRONTEND=noninteractive apt-get upgrade \ +list_file="/var/lib/apt/lists/archive.turnkeylinux.org_debian_dists_*" +[[ -n "$list_file" ]] || apt-get update + +apt-get upgrade \ --autoremove --with-new-pkgs -y \ -o DPkg::Options::=--force-confdef \ -o DPkg::Options::=--force-confold - -INSTALLED=$(dpkg-query --showformat='${Package} ${Status}\n' -W 'linux-image-[0-9].*' | grep "ok installed" | sed 's/ .*//') -CURRENT=$(ls -l /vmlinuz | awk '{print $11}' | sed 's|boot/vmlinuz-|linux-image-|') -for KERNEL in $INSTALLED; do - [ "$KERNEL" == "$CURRENT" ] && continue - DEBIAN_FRONTEND=noninteractive apt-get -y purge $KERNEL -done diff --git a/patches/clean-old-kernels/conf b/patches/clean-old-kernels/conf new file mode 100755 index 0000000..e69a965 --- /dev/null +++ b/patches/clean-old-kernels/conf @@ -0,0 +1,17 @@ +#!/bin/bash -ex + +export DEBIAN_FRONTEND=noninteractive + +old_kernels=() +installed_kernels=$(dpkg -l | grep '^ii *linux-image-[0-9]' | awk '{print $2}') +current_vmlinuz=$(find / -maxdepth 1 -name 'vmlinuz' -exec readlink {} \;) +current_kernel="linux-image-${current_vmlinuz#boot/vmlinuz-}" +for kernel in $installed_kernels; do + if [[ "$kernel" == "$current_kernel" ]]; then + continue + else + old_kernels+=("$kernel") + fi +done + +apt-get -y purge "${old_kernels[@]}" || true From 5818d7a553ac298ffa4845b7ccf9cb68f2827b23 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 5 Sep 2024 16:23:29 +1000 Subject: [PATCH 04/10] Add new bt-bugfix-single script --- bt-bugfix-single | 248 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100755 bt-bugfix-single diff --git a/bt-bugfix-single b/bt-bugfix-single new file mode 100755 index 0000000..0e3a855 --- /dev/null +++ b/bt-bugfix-single @@ -0,0 +1,248 @@ +#!/bin/bash -e +# Copyright (c) 2024 TurnKey GNU/Linux - https://www.turnkeylinux.org +# +# This file is part of buildtasks. +# +# Buildtasks is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. + +export DEBIAN_STABLE=bookworm +this_arch=$(dpkg --print-architecture) +export OS_ARCH="$this_arch" + +fatal() { echo "FATAL [$(basename "$0")]: $*" 1>&2; exit 1; } +warning() { echo "WARNING [$(basename "$0")]: $*" 1>&2; } +info() { echo "INFO [$(basename "$0")]: $*"; } + +usage() { +cat< "$rootfs/etc/turnkey_version" +update_release=/tmp/update-release-pkg +mkdir "$update_release/debs" +"$BT/bin/generate-release-deb" "$new_changelog" "$update_release/debs/" +cat > "$update_release/conf" < "$O/$new_name.manifest" + "$BT/bin/generate-buildenv" iso "$appname" > "$O/$new_name.iso.buildenv" + if [[ -e "$BT_PROFILES/$appname" ]]; then + mkdir -p "$O/$new_name.tklbam" + export PROFILES_CONF=$BT_PROFILES + "$BT/bin/generate-tklbam-profile" "$O/$new_name.iso" "$O/$new_name.tklbam" + else + fatal "tklbam profile not found: $BT_PROFILES/$appname" + fi + "$BT/bin/iso-publish" "$BT_ISOS/$new_name.iso" +fi +_finish From 94dfa50b50d3a9106279ed6731c6560d1687d19f Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 5 Sep 2024 07:17:03 +0000 Subject: [PATCH 05/10] Add canvas v18.1 patch --- patches/canvas-18.0-bookworm-amd64/changelog | 416 ++++++++++++++++++ patches/canvas-18.0-bookworm-amd64/conf | 12 + .../overlay/etc/tmpfiles.d/passenger.conf | 1 + .../overlay/usr/lib/inithooks/bin/canvas.py | 133 ++++++ 4 files changed, 562 insertions(+) create mode 100644 patches/canvas-18.0-bookworm-amd64/changelog create mode 100755 patches/canvas-18.0-bookworm-amd64/conf create mode 100644 patches/canvas-18.0-bookworm-amd64/overlay/etc/tmpfiles.d/passenger.conf create mode 100755 patches/canvas-18.0-bookworm-amd64/overlay/usr/lib/inithooks/bin/canvas.py diff --git a/patches/canvas-18.0-bookworm-amd64/changelog b/patches/canvas-18.0-bookworm-amd64/changelog new file mode 100644 index 0000000..cf4b471 --- /dev/null +++ b/patches/canvas-18.0-bookworm-amd64/changelog @@ -0,0 +1,416 @@ +turnkey-canvas-18.1 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable (prod branch), Canvas RCE API and required + dependencies. Canvas installed from upstream git repo. + + * Update Ruby (3.1.6). + + * Update bundler to 2.5.10 - as per "Production Start" doc. + + * Disable Apache mod_evasie for Canvas - part of #1965. + + * Run switchman_inst_jobs:install:migrations - closes #1965. + + * Update GEM_PATH in Apache conf - didn't seem to be causing issues, but + better for it to be correct path. + + * Apply fix so passenger-status works - useful for debugging. + + * Configuration console (confconsole) - v2.1.6: + - Let's Encrypt/Dehydrated - bugfix cron failure - closes #1962. + - General dehydrated-wrapper code cleanup - now passes shellcheck. + + * Web management console (webmin): + - Include webmin-logviewer module by default - closes #1866. + - Replace webmin-shell with webmin-xterm module by default - closes #1904. + + * Reduce log noise by creating ntpsec log dir - closes #1952. + + -- Jeremy Davis Sat, 06 Jul 2024 11:31:36 +0000 + +turnkey-canvas-18.0 (1) turnkey; urgency=low + + * Update Ruby (3.1.4). + [Zhenya Hvorostian ] + + * Update yarn (1.22.19). + [Zhenya Hvorostian ] + + * Install latest Canvas LTS stable (prod branch), Canvas RCE API and required dependencies. Canvas + installed from upstream git repo. + [Zhenya Hvorostian ] + + * Ensure hashfile includes URL to public key - closes #1864. + + * Include webmin-logviewer module by default - closes #1866. + + * Upgraded base distribution to Debian 12.x/Bookworm. + + * Configuration console (confconsole): + - Support for DNS-01 Let's Encrypt challenges. + [ Oleh Dmytrychenko github: @NitrogenUA ] + - Support for getting Let's Encrypt cert via IPv6 - closes #1785. + - Refactor network interface code to ensure that it works as expected and + supports more possible network config (e.g. hotplug interfaces & wifi). + - Show error message rather than stacktrace when window resized to + incompatable resolution - closes #1609. + [ Stefan Davis ] + - Bugfix exception when quitting configuration of mail relay. + [ Oleh Dmytrychenko github: @NitrogenUA ] + - Improve code quality: implement typing, fstrings and make (mostly) PEP8 + compliant. + [Stefan Davis & Jeremy Davis + + * Firstboot Initialization (inithooks): + - Refactor start up (now hooks into getty process, rather than having it's + own service). + [ Stefan Davis ] + - Refactor firstboot.d/01ipconfig (and 09hostname) to ensure that hostname + is included in dhcp info when set via inithooks. + - Package turnkey-make-ssl-cert script (from common overlay - now packaged + as turnkey-ssl). Refactor relevant scripts to leverage turnkey-ssl. + - Refactor run script - use bashisms and general tidying. + - Show blacklisted password characters more nicely. + - Misc packaging changes/improvements. + - Support returning output from MySQL - i.e. support 'SELECT'. (Only + applies to apps that include MySQL/MariaDB). + + * Web management console (webmin): + - Upgraded webmin to v2.105. + - Removed stunnel reverse proxy (Webmin hosted directly now). + - Ensure that Webmin uses HTTPS with default cert + (/etc/ssl/private/cert.pem). + - Disabled Webmin Let's Encrypt (for now). + + * Web shell (shellinabox): + - Completely removed in v18.0 (Webmin now has a proper interactive shell). + + * Backup (tklbam): + - Ported dependencies to Debian Bookworm; otherwise unchanged. + + * Security hardening & improvements: + - Generate and use new TurnKey Bookworm keys. + - Automate (and require) default pinning for packages from Debian + backports. Also support non-free backports. + + * IPv6 support: + - Adminer (only on LAMP based apps) listen on IPv6. + - Nginx/NodeJS (NodeJS based apps only) listen on IPv6. + + * Misc bugfixes & feature implementations: + - Remove rsyslog package (systemd journal now all that's needed). + - Include zstd compression support. + - Enable new non-free-firmware apt repo by default. + - Improve turnkey-artisan so that it works reliably in cron jobs (only + Laravel based LAMP apps). + + * Use PostgreSQL v15 (from debian repos). + [ Stefan Davis ] + + * Upstream/Debian Adminer update - closes #1758. + [ Stefan Davis ] + + * Set mod_evasive log location - makes debugging easier. + [ Jeremy Davis ] + + * Include and enable mod_evasive and mod_security2 by default in Apache. + [ Stefan Davis ] + + -- Jeremy Davis Mon, 05 Feb 2024 07:00:32 +0000 + +turnkey-canvas-17.1 (1) turnkey; urgency=low + + * Updated all Debian packages to latest. + [ autopatched by buildtasks ] + + * Patched bugfix release. Closes #1734. + [ autopatched by buildtasks ] + + -- Jeremy Davis Tue, 21 Feb 2023 02:54:06 +0000 + +turnkey-canvas-17.0 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable (prod branch) and required dependencies. Canvas + installed from upstream git repo. + [ Zhenya Hvorostian & + Anton Pyrogovskyi ] + + * Use LTS Node version already installed by common (currently 18.12.1). + + * Install yarn version recommended by upstream (currently 1.22.11). + + * Note: Please refer to turnkey-core's 17.0 changelog for changes common to + all appliances. Here we only describe changes specific to this appliance. + + -- Anton Pyrogovskyi Fri, 16 Dec 2022 19:36:39 +0100 + +turnkey-canvas-16.2 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable and required dependencies. Canvas + installed from upstream git repo. + + * Note: Please refer to turnkey-core's 16.1 changelog for changes common to + all appliances. Here we only describe changes specific to this appliance. + + -- Jeremy Davis Thu, 27 May 2021 13:20:31 +1000 + +turnkey-canvas-16.1 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable from upstream git repo. + [ Jeremy Davis & + Zhenya Hvorostian ] + + * Fix/improve domain setting inithooks (untracked). + [ Zhenya Hvorostian & + Jeremy Davis ] + + * Fix canvas_init script (untracked issue with false regex). + [ Zhenya Hvorostian ] + + * Install yarn via common conf/overlay (remove redundant bits from this + repo). + [ Jeremy Davis & + Zhenya Hvorostian ] + + * Improved/more explicit mail settings. + [ Zhenya Hvorostian ] + + * Make cache_store.yml permissions stricter. + [ Zhenya Hvorostian ] + + * Improve/finalize RCE API integration and fix 500 errors. (Note previous + releases had RCE API integration, but this finalises it). Closes #1319 & + #1362. + [ Zhenya Hvorostian ] + + * Remove redundant "stand-alone" Passenger service (and service calls). This + was starting Canvas via Passenger under Nginx, rather than Apache under + some circusmtance. Closes #1495. + + -- Jeremy Davis Thu, 06 Aug 2020 12:11:52 +1000 + +turnkey-canvas-16.0 (1) turnkey; urgency=low + + * Explcitly disable TLS<1.2 (i.e. SSLv3, TLSv1, TLSv1.1). (v15.x + TurnKey releases supported TLS 1.2, but could fallback as low as TLSv1). + + * Update SSL/TLS cyphers to provide "Intermediate" browser/client support + (suitable for "General-purpose servers with a variety of clients, + recommended for almost all systems"). As provided by Mozilla via + https://ssl-config.mozilla.org/. + + * Install latest Canvas LTS stable from upstream git repo. + + * Repaired Canvas RCE API service for running with a domain. + + * Update included NodeJS to 10.x (10.20.1). + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Zhenya Hvorostian Wed, 13 May 2020 20:12:15 +0300 + +turnkey-canvas-15.3 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable from upstream git repo. + + * Include Canvas RCE API service run with Passenger - closes #1319. + + * Update included NodeJS to 10.x (10.15.3). + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Zhenya Hvorostian Thu, 06 June 2019 18:38:42 +0300 + +turnkey-canvas-15.2 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable from upstream git repo. + + * Include updated yarn gpg key. + + * Include Canvas logrotate.d script - closes #1279. + [ Zhenya Hvorostian ] + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Jeremy Davis Tue, 05 Feb 2019 17:22:41 +1100 + +turnkey-canvas-15.1 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable from upstream git repo. + + * Update yarn version to 1.10.1. + + * Secure yarn apt repo. + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Zhenya Hvorostian Fri, 23 Nov 2018 11:48:21 +0300 + +turnkey-canvas-15.0 (1) turnkey; urgency=low + + * Install latest Canvas LTS stable from upstream git repo. + + * Update yarn version to latest. + + * Change Node.js version to 8.12.0. + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Zhenya Hvorostian Sun, 24 Sep 2018 21:33:10 +0300 + +turnkey-canvas-14.2 (1) turnkey; urgency=low + + * Latest version of Canvas installed. + + * Now includes LTS Node.js (v6.11.1). + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Anton Pyrogovskyi Mon, 31 Jul 2017 10:07:26 +0200 + +turnkey-canvas-14.1 (2) turnkey; urgency=low + + * Canvas: + + - Responsiveness when running on AWS is improved [#583]. + - Now deployed on Passenger Standalone instead of Apache. + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Anton Pyrogovskyi Mon, 02 May 2016 18:44:51 +0200 + +turnkey-canvas-14.1 (1) turnkey; urgency=low + + * Canvas: + + - Bugs related to DelayedJobs (fixes #516). + - Improved regen-rails-secrets inithook [#591]. + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Stefan Davis Mon, 11 Jan 2016 00:40:20 +0000 + +turnkey-canvas-14.0 (1) turnkey; urgency=low + + * Canvas: + + - Changed database adapter to postgresql (support for MySQL dropped). + + * Upstream source component versions: + + canvas-lms stable (git branch) + + * Hardened default SSL settings + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Anton Pyrogovskyi Sun, 6 Sep 2015 13:15:27 +0300 + +turnkey-canvas-13.0 (1) turnkey; urgency=low + + * Canvas: + + - Changed database adapter to mysql2 (recommended for ruby 1.9) + - Replaced ruby-enterprise with stock ruby from Debian [#102]. + - Redis installed from Debian archive (backport no longer needed). + + * Build related changes: + + - Bugfix: don't delete /usr/local/src/node* + - Install activesupport 2.3.18 and rerun bundle install on failure [#109]. + - Comment out problematic migration [#110]. + - Bugfixes for package transitions [#58, #59]. + + * Upstream source component versions: + + canvas-lms stable (git branch) + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Alon Swartz Thu, 10 Oct 2013 17:05:35 +0300 + +turnkey-canvas-12.1 (1) turnkey; urgency=low + + * Canvas: + + - Upgraded to latest version of Canvas - cloned from upstream git repo + as recommended providing an easy upgrade path. + - Fixed automated jobs daemon configuration and enabled (bugfix). + - Set outgoing mail defaults to tls, disable for local postfix (bugfix). + - Installing and pinning latest version of redis-server from backports. + - Upgraded ruby version (required in latest version). + + * NodeJS related: + + - Latest version of node will be installed at build time, with a source + symlink created at /usr/local/src/node. + - Node binaries added to path via symlinking in /usr/local/bin. + - Latest NPM module versions installed at build time. + + * Upstream source component versions: + + canvas-lms stable (git branch) + rubyenterprise 1.8.7-2012.02_i386_ubuntu10.04 + 1.8.7-2012.02_amd64_debian6 + ruby 1.9.3-p327 + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Alon Swartz Sun, 07 Apr 2013 08:00:00 +0200 + +turnkey-canvas-12.0 (1) turnkey; urgency=low + + * Initial public release of TurnKey Canvas. + + * Canvas related: + + - Canvas LMS and nodejs installed from latest upstream version. Redis + server installed from Squeeze backports as required. + - Set Canvas admin password and email on firstboot (convenience, security). + - Set Canvas domain to serve on first boot (convenience). + - Pre-configured to use MySQL (recommended for production). + - Includes Canvas automated jobs daemon initscript (disabled by default). + - Includes Apache pre-configured with passenger support, with SSL support + out of the box (performance, security). + + * Regenerates all secrets during installation / firstboot (security). + + * MySQL related: + + - Set MySQL root password on firstboot (convenience, security). + - Force MySQL to use Unicode/UTF8. + + * Includes postfix MTA (bound to localhost) for sending of email (e.g. + password recovery). Also includes webmin postfix module for convenience. + + * Major component versions + + canvas-lms revision db0034c (upstream archive) + + rails 2.3.14 + ruby-enterprise 1.8.7-2012.02 + ruby 4.5 + ruby-dev 4.5 + + apache2 2.2.16-6+squeeze7 + mysql-server 5.1.63-0+squeeze1 + redis-server 2.4.15-1~bpo60+2 (backported package) + build-essential 11.5 + imagemagick 8:6.6.0.4-3+squeeze3 + + * Note: Please refer to turnkey-core's changelog for changes common to all + appliances. Here we only describe changes specific to this appliance. + + -- Alon Swartz Wed, 01 Aug 2012 08:00:00 +0200 + diff --git a/patches/canvas-18.0-bookworm-amd64/conf b/patches/canvas-18.0-bookworm-amd64/conf new file mode 100755 index 0000000..5c3dc02 --- /dev/null +++ b/patches/canvas-18.0-bookworm-amd64/conf @@ -0,0 +1,12 @@ +#!/bin/bash -e + +passenger_conf=/etc/apache2/mods-available/passenger.conf +sed -i "/^PassengerStartTimeout/ s| .*| 360|" "$passenger_conf" +echo "PassengerInstanceRegistryDir /run/passenger-instreg" >> "$passenger_conf" + +conf_dir=/var/www/canvas/config +sed -i "/worker_max_memory_usage:/ s|:.*|:1073741824|" "$conf_dir/security.yml" +sed -i "/lti_iss:/ S|:.*|: \"https://www.example.com\"" "$conf_dir/security.yml" + +py3clean / +yarn cache clean diff --git a/patches/canvas-18.0-bookworm-amd64/overlay/etc/tmpfiles.d/passenger.conf b/patches/canvas-18.0-bookworm-amd64/overlay/etc/tmpfiles.d/passenger.conf new file mode 100644 index 0000000..d8f4e76 --- /dev/null +++ b/patches/canvas-18.0-bookworm-amd64/overlay/etc/tmpfiles.d/passenger.conf @@ -0,0 +1 @@ +d /run/passenger-instreg 0755 root root - diff --git a/patches/canvas-18.0-bookworm-amd64/overlay/usr/lib/inithooks/bin/canvas.py b/patches/canvas-18.0-bookworm-amd64/overlay/usr/lib/inithooks/bin/canvas.py new file mode 100755 index 0000000..fe3fbd1 --- /dev/null +++ b/patches/canvas-18.0-bookworm-amd64/overlay/usr/lib/inithooks/bin/canvas.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +"""Set Canvas admin password, email and domain to serve + +Option: + --pass= unless provided, will ask interactively + --email= unless provided, will ask interactively + --domain= unless provided, will ask interactively + DEFAULT=www.example.com +""" + +import sys +import getopt +import hashlib +import random +import string +import psycopg2 +import subprocess + +from libinithooks import inithooks_cache +from libinithooks.dialog_wrapper import Dialog + +DEFAULT_DOMAIN = "www.example.com" + + +def usage(s=None): + if s: + print("Error:", s, file=sys.stderr, **kwargs) + print(f"Syntax: {sys.argv[0]} [options]", file=sys.stderr) + print(__doc__, file=sys.stderr) + sys.exit(1) + + +def main(): + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], "h", + ['help', 'pass=', 'email=', 'domain=']) + except getopt.GetoptError as e: + usage(e) + + email = "" + domain = "" + password = "" + for opt, val in opts: + if opt in ('-h', '--help'): + usage() + elif opt == '--pass': + password = val + elif opt == '--email': + email = val + elif opt == '--domain': + domain = val + + if not password: + d = Dialog('TurnKey Linux - First boot configuration') + password = d.get_password( + "Canvas Password", + "Enter new password for the Canvas 'admin' account.") + + if not email: + if 'd' not in locals(): + d = Dialog('TurnKey Linux - First boot configuration') + + email = d.get_email( + "Canvas Email", + "Enter email address for the Canvas 'admin' account.", + "admin@example.com") + + inithooks_cache.write('APP_EMAIL', email) + + if not domain: + if 'd' not in locals(): + d = Dialog('TurnKey Linux - First boot configuration') + + domain = d.get_input( + "Canvas Domain", + "Enter the domain to serve Canvas.", + DEFAULT_DOMAIN) + + if domain == "DEFAULT": + domain = DEFAULT_DOMAIN + + inithooks_cache.write('APP_DOMAIN', domain) + + salt = "".join(random.choice(string.ascii_letters) for line in range(20)) + hash = password + salt + for i in range(20): + hash = hashlib.sha512(hash.encode('utf-8')).hexdigest() + + access_token = "".join(random.choice(string.ascii_letters) + for line in range(20)) + + conn = psycopg2.connect("dbname=canvas_production user=root") + c = conn.cursor() + c.execute('UPDATE users SET name=%s, sortable_name=%s WHERE id=1;', + (email, email)) + c.execute('UPDATE pseudonyms SET unique_id=%s, crypted_password=%s, password_salt=%s, single_access_token=%s WHERE user_id=1;', + (email, hash, salt, access_token)) + c.execute('UPDATE communication_channels SET path=%s WHERE id=1;', + (email, )) + conn.commit() + c.close() + conn.close() + + config = "/var/www/canvas/config/outgoing_mail.yml" + subprocess.run(["sed", "-ri", + f's|domain:.*|domain: "{domain}"|', + config]) + subprocess.run(["sed", "-ri", + f's|outgoing_address:.*|outgoing_address: "{email}"|', + config]) + + config = "/var/www/canvas/config/dynamic_settings.yml" + subprocess.run(["sed", "-ri", + f's|app-host:.*|app-host: "{domain}:3000"|', + config]) + + config = "/var/www/canvas/config/domain.yml" + subprocess.run(["sed", "-ri", + f's|domain:.*|domain: "{domain}"|', + config]) + + config = "/var/www/canvas/config/security.yml" + subprocess.run(["sed", "-ri", + f's|lti_iss:.*|lti_iss: "https://{domain}"|', + config]) + + print("Restarting services; please wait...") + for service in ['canvas_init', 'apache2']: + subprocess.run(['systemctl', 'restart', service]) + + +if __name__ == "__main__": + main() From 1328c282008046698fdacf19a130359eb23b95a0 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 6 Sep 2024 04:16:33 +0000 Subject: [PATCH 06/10] Refactoring, improvements & bugfixes --- bt-bugfix-single | 132 ++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/bt-bugfix-single b/bt-bugfix-single index 0e3a855..3291188 100755 --- a/bt-bugfix-single +++ b/bt-bugfix-single @@ -11,17 +11,18 @@ export DEBIAN_STABLE=bookworm this_arch=$(dpkg --print-architecture) export OS_ARCH="$this_arch" +APP_NAME=$(basename "$0") -fatal() { echo "FATAL [$(basename "$0")]: $*" 1>&2; exit 1; } -warning() { echo "WARNING [$(basename "$0")]: $*" 1>&2; } -info() { echo "INFO [$(basename "$0")]: $*"; } +fatal() { echo "FATAL [$APP_NAME]: $*" 1>&2; exit 1; } +warning() { echo "WARNING [$APP_NAME]: $*" 1>&2; } +info() { echo "INFO [$APP_NAME]: $*"; } usage() { cat</dev/null || true + done + fi + if mount | grep -q "$rootfs"; then + fatal "$rootfs not unmounted." + fi } _clean_dirs() { - if ! (mount | grep -q "$(basename "$rootfs")"); then - rm -rf "${O:?}/${rootfs:?}" - rm -rf "${O:?}/${cdroot:?}" - else - fatal "$rootfs not unmounted." - fi + _umount + rm -rf "${O:?}/${rootfs:?}" + rm -rf "${O:?}/${cdroot:?}" } -_clean_isos() { +_clean_iso() { local iso=$1 rm -rf "${O:?}/${iso:?}" rm -rf "${O:?}/${iso:?}.hash" @@ -90,14 +98,25 @@ _clean_isos() { _start() { info "cleaning relevant assets before build" - _clean_isos "$new_isofile" - _clean_dirs - if [[ -z "$local_iso" ]]; then + _clean_iso "$new_isofile" + if [[ -z "$local_rootfs" ]] && [[ -z "$local_iso" ]]; then + _clean_dirs _clean_isos "$isofile" - else - warning "--local-iso set" + elif [[ -n "$local_rootfs" ]]; then + msg="--rootfs set" + if [[ ! -d "${O:?}/$rootfs" ]] \ + || [[ ! -d "${O:?}/$cdroot" ]]; then + fatal "$msg and rootfs &/or cdroot do not exist in $BT_ISOS" + else + _umount + warning "$msg - using existing rootfs & cdroot dirs" + fi + elif [[ -n "$local_iso" ]]; then + msg="--local-iso set" if [[ ! -f "${O:?}/$isofile" ]] || [[ ! -f "${O:?}/$isofile.hash" ]]; then - fatal "iso &/or hash files do not exist in $BT_ISOS" + fatal "$msg & iso file does not exist in $BT_ISOS" + else + warning "$msg - using existing iso & hash file" fi fi } @@ -107,10 +126,13 @@ _finish() { if [[ -z "$BT_DEBUG" ]]; then info "cleaning up build assets" _clean_dirs + rm -rf "$TMP_DIR" + else + warning "BT_DEBUG set - rootfs, cdroot & $TMP_DIR not removed" fi } -unset appver publish sec_updates all_updates local_iso finish +unset appver publish sec_updates all_updates local_iso local_rootfs while [[ "$1" != "" ]]; do case $1 in --help|-h) usage;; @@ -120,6 +142,7 @@ while [[ "$1" != "" ]]; do --updates) all_updates="yes" sec_updates="yes";; --local-iso) local_iso="yes";; + --rootfs) local_rootfs="yes";; *) if [[ -n "$appver" ]]; then usage else @@ -132,7 +155,7 @@ done [[ -z "$BT_DEBUG" ]] || set -x [[ -n "$appver" ]] || usage -info "Parse and validate args, opts and required build variables" +info "Parsing and validating args, opts and required build variables" bt_bugfix_path=$(readlink -f "$0") bt_dir=$(dirname "$bt_bugfix_path") export BT="$bt_dir" @@ -171,6 +194,7 @@ if [[ "$publish" == "yes" ]]; then [[ -n "$BT_PUBLISH_IMGS" ]] || fatal "BT_PUBLISH_IMGS not set" [[ -n "$BT_PUBLISH_META" ]] || fatal "BT_PUBLISH_META not set" [[ -n "$BT_PUBLISH_PROFILES" ]] || fatal "BT_PUBLISH_PROFILES not set" + [[ -n "$local_roofs" ]] || fatal "--rootfs conflicts with --publish" [[ -n "$local_iso" ]] || fatal "--local-iso conflicts with --publish" else warning "--publish was not set" @@ -181,53 +205,57 @@ fi if [[ ! -d "$ver_patch" ]]; then fatal "Patch not found: $ver_patch" elif [[ ! -f "$ver_patch/changelog" ]]; then - fatal "New changelog not found in patch directory" -elif ! grep "^$new_sec_pkg " <(head -1 "$ver_patch/changelog"); then + fatal "$ver_patch/changelog not found" +elif ! grep -q "^$new_sec_pkg " <(head -1 "$ver_patch/changelog"); then fatal "Patch changelog has incorrect version (should be $new_sec_pkg)" fi O=$BT_ISOS mkdir -p "$O" _start - -"$BT/bin/iso-download" "$O" "$BT_VERSION" "$appname" -"$BT/bin/iso-verify" "$O" "$BT_VERSION" "$appname" - -info "Unpacking iso and setting up as chroot" cd "$O" -tklpatch-extract-iso "$isofile" + +if [[ -z "$local_rootfs" ]]; then + "$BT/bin/iso-download" "$O" "$BT_VERSION" "$appname" + "$BT/bin/iso-verify" "$O" "$BT_VERSION" "$appname" + info "Unpacking iso and setting up as chroot" + tklpatch-extract-iso "$isofile" +fi mount --bind --make-rslave /proc "$rootfs/proc" mount --bind --make-rslave /sys "$rootfs/sys" mount --bind --make-rslave /dev "$rootfs/dev" mount --bind --make-rslave /run "$rootfs/run" -info "Updating appliance rootfs and applying patch" +TMP_DIR=$(mktemp -d /tmp/$APP_NAME.XXXXX) + +info "Updating appliance rootfs & applying patch" echo "$new_name" > "$rootfs/etc/turnkey_version" -update_release=/tmp/update-release-pkg -mkdir "$update_release/debs" -"$BT/bin/generate-release-deb" "$new_changelog" "$update_release/debs/" -cat > "$update_release/conf" < "$release_patch/conf" </dev/null \ + || warning "package: $old_sec_pkg not found" EOF -chmod +x "$update_release/conf" -tklpatch-apply "$update_release" -rm -rf "$update_release" +chmod +x "$release_patch/conf" +tklpatch-apply "$rootfs" "$release_patch" +rm -rf "$release_patch" if [[ -n "$all_updates" ]]; then tklpatch-apply "$rootfs" "$BT/patches/apt-upgrade" elif [[ -n "$sec_updates" ]]; then - turnkey-install-security-updates + fab-chroot "$rootfs" turnkey-install-security-updates fi -info "Cleaning up rootfs and rebuilding new ISO." +info "Building new ISO." tklpatch-apply "$rootfs" "$BT/patches/clean-old-kernels" "$BT/bin/rootfs-cleanup" "$rootfs" "$BT/bin/aptconf-tag" "$rootfs" iso +rm -rf "$rootfs/tmp/*" +_umount tklpatch-prepare-cdroot "$rootfs" "$cdroot" TKLPATCH_ISOLABEL=${appname} tklpatch-geniso "$cdroot" "$new_isofile" From dcaa49ecc8e0c6fdccd4e26dedc328e048cff97e Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Fri, 6 Sep 2024 14:24:28 +1000 Subject: [PATCH 07/10] Minor linting fixes & a typo fix --- bt-bugfix-single | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bt-bugfix-single b/bt-bugfix-single index 3291188..27f79f5 100755 --- a/bt-bugfix-single +++ b/bt-bugfix-single @@ -1,8 +1,8 @@ #!/bin/bash -e # Copyright (c) 2024 TurnKey GNU/Linux - https://www.turnkeylinux.org -# +# # This file is part of buildtasks. -# +# # Buildtasks is free software; you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published by the # Free Software Foundation; either version 3 of the License, or (at your @@ -101,7 +101,7 @@ _start() { _clean_iso "$new_isofile" if [[ -z "$local_rootfs" ]] && [[ -z "$local_iso" ]]; then _clean_dirs - _clean_isos "$isofile" + _clean_iso "$isofile" elif [[ -n "$local_rootfs" ]]; then msg="--rootfs set" if [[ ! -d "${O:?}/$rootfs" ]] \ @@ -186,7 +186,6 @@ rootfs=$name.rootfs cdroot=$name.cdroot old_sec_pkg=turnkey-${appname}-${appversion} new_sec_pkg=turnkey-${appname}-${new_appversion} -new_changelog=$new_name.changelog ver_patch=$BT/patches/${appname}-${BT_VERSION} [[ -n "$BT_ISOS" ]] || fatal "BT_ISO not set" @@ -227,7 +226,7 @@ mount --bind --make-rslave /sys "$rootfs/sys" mount --bind --make-rslave /dev "$rootfs/dev" mount --bind --make-rslave /run "$rootfs/run" -TMP_DIR=$(mktemp -d /tmp/$APP_NAME.XXXXX) +TMP_DIR=$(mktemp -d /tmp/"$APP_NAME".XXXXX) info "Updating appliance rootfs & applying patch" echo "$new_name" > "$rootfs/etc/turnkey_version" @@ -238,7 +237,7 @@ cat > "$release_patch/conf" </dev/null \ - || warning "package: $old_sec_pkg not found" + || echo "WARNING package: $old_sec_pkg not found" >&2 EOF chmod +x "$release_patch/conf" tklpatch-apply "$rootfs" "$release_patch" From 3a0bb20294e8b552ac09408d73931d47b449e862 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 12 Sep 2024 07:38:19 +0000 Subject: [PATCH 08/10] Disable mod_evasive and Apache private tmp --- patches/canvas-18.0-bookworm-amd64/conf | 2 ++ .../etc/systemd/system/apache2.service.d/override.conf | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 patches/canvas-18.0-bookworm-amd64/overlay/etc/systemd/system/apache2.service.d/override.conf diff --git a/patches/canvas-18.0-bookworm-amd64/conf b/patches/canvas-18.0-bookworm-amd64/conf index 5c3dc02..b1f0908 100755 --- a/patches/canvas-18.0-bookworm-amd64/conf +++ b/patches/canvas-18.0-bookworm-amd64/conf @@ -1,5 +1,7 @@ #!/bin/bash -e +a2dismod evasive + passenger_conf=/etc/apache2/mods-available/passenger.conf sed -i "/^PassengerStartTimeout/ s| .*| 360|" "$passenger_conf" echo "PassengerInstanceRegistryDir /run/passenger-instreg" >> "$passenger_conf" diff --git a/patches/canvas-18.0-bookworm-amd64/overlay/etc/systemd/system/apache2.service.d/override.conf b/patches/canvas-18.0-bookworm-amd64/overlay/etc/systemd/system/apache2.service.d/override.conf new file mode 100644 index 0000000..bd88c98 --- /dev/null +++ b/patches/canvas-18.0-bookworm-amd64/overlay/etc/systemd/system/apache2.service.d/override.conf @@ -0,0 +1,6 @@ +[Service] +# Disable Apache private temp so the detailed passenger error html files are +# saved directly to the root of /tmp - otherwise they are saved to +# /tmp/systemd-private-xxx.xxx-apache2.service-XXXXXX/tmp/passenger-error-xxxxx.html +# making them hard to find for the uninitiated +PrivateTmp=false From 9d4811bd753fecbcce256846c7dbe1a48bc75f40 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 12 Sep 2024 07:39:01 +0000 Subject: [PATCH 09/10] Update canvas changelog - and remove errant whitespace --- patches/canvas-18.0-bookworm-amd64/changelog | 42 +++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/patches/canvas-18.0-bookworm-amd64/changelog b/patches/canvas-18.0-bookworm-amd64/changelog index cf4b471..198e142 100644 --- a/patches/canvas-18.0-bookworm-amd64/changelog +++ b/patches/canvas-18.0-bookworm-amd64/changelog @@ -1,20 +1,14 @@ turnkey-canvas-18.1 (1) turnkey; urgency=low - * Install latest Canvas LTS stable (prod branch), Canvas RCE API and required - dependencies. Canvas installed from upstream git repo. + * Update to Canvas security.yml config: + - delayed_jobs.yml - bump worker_max_memory_usage memory allowance + - closes #1979. + - security.yml - set domain for lti_iss - untracked bug. - * Update Ruby (3.1.6). + * Disable Apache mod_evasive for Canvas - part of #1965. - * Update bundler to 2.5.10 - as per "Production Start" doc. - - * Disable Apache mod_evasie for Canvas - part of #1965. - - * Run switchman_inst_jobs:install:migrations - closes #1965. - - * Update GEM_PATH in Apache conf - didn't seem to be causing issues, but - better for it to be correct path. - - * Apply fix so passenger-status works - useful for debugging. + * Disable Apache PrivateTmp so passenger error files can be found where + passenger reports they can be found (i.e. /tmp) - untracked bug. * Configuration console (confconsole) - v2.1.6: - Let's Encrypt/Dehydrated - bugfix cron failure - closes #1962. @@ -26,7 +20,7 @@ turnkey-canvas-18.1 (1) turnkey; urgency=low * Reduce log noise by creating ntpsec log dir - closes #1952. - -- Jeremy Davis Sat, 06 Jul 2024 11:31:36 +0000 + -- Jeremy Davis Thu, 12 Sep 2024 07:35:16 +0000 turnkey-canvas-18.0 (1) turnkey; urgency=low @@ -204,7 +198,7 @@ turnkey-canvas-16.0 (1) turnkey; urgency=low * Repaired Canvas RCE API service for running with a domain. * Update included NodeJS to 10.x (10.20.1). - + * Note: Please refer to turnkey-core's changelog for changes common to all appliances. Here we only describe changes specific to this appliance. @@ -217,7 +211,7 @@ turnkey-canvas-15.3 (1) turnkey; urgency=low * Include Canvas RCE API service run with Passenger - closes #1319. * Update included NodeJS to 10.x (10.15.3). - + * Note: Please refer to turnkey-core's changelog for changes common to all appliances. Here we only describe changes specific to this appliance. @@ -231,7 +225,7 @@ turnkey-canvas-15.2 (1) turnkey; urgency=low * Include Canvas logrotate.d script - closes #1279. [ Zhenya Hvorostian ] - + * Note: Please refer to turnkey-core's changelog for changes common to all appliances. Here we only describe changes specific to this appliance. @@ -240,27 +234,27 @@ turnkey-canvas-15.2 (1) turnkey; urgency=low turnkey-canvas-15.1 (1) turnkey; urgency=low * Install latest Canvas LTS stable from upstream git repo. - + * Update yarn version to 1.10.1. * Secure yarn apt repo. - + * Note: Please refer to turnkey-core's changelog for changes common to all appliances. Here we only describe changes specific to this appliance. - + -- Zhenya Hvorostian Fri, 23 Nov 2018 11:48:21 +0300 turnkey-canvas-15.0 (1) turnkey; urgency=low * Install latest Canvas LTS stable from upstream git repo. - + * Update yarn version to latest. - + * Change Node.js version to 8.12.0. * Note: Please refer to turnkey-core's changelog for changes common to all appliances. Here we only describe changes specific to this appliance. - + -- Zhenya Hvorostian Sun, 24 Sep 2018 21:33:10 +0300 turnkey-canvas-14.2 (1) turnkey; urgency=low @@ -343,7 +337,7 @@ turnkey-canvas-12.1 (1) turnkey; urgency=low * Canvas: - - Upgraded to latest version of Canvas - cloned from upstream git repo + - Upgraded to latest version of Canvas - cloned from upstream git repo as recommended providing an easy upgrade path. - Fixed automated jobs daemon configuration and enabled (bugfix). - Set outgoing mail defaults to tls, disable for local postfix (bugfix). From 1992582232288cc52ff567be791058321ecfb804 Mon Sep 17 00:00:00 2001 From: Jeremy Davis Date: Thu, 12 Sep 2024 10:04:12 +0000 Subject: [PATCH 10/10] Fix mistake in patch conf --- patches/canvas-18.0-bookworm-amd64/conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patches/canvas-18.0-bookworm-amd64/conf b/patches/canvas-18.0-bookworm-amd64/conf index b1f0908..e141ab3 100755 --- a/patches/canvas-18.0-bookworm-amd64/conf +++ b/patches/canvas-18.0-bookworm-amd64/conf @@ -7,7 +7,7 @@ sed -i "/^PassengerStartTimeout/ s| .*| 360|" "$passenger_conf" echo "PassengerInstanceRegistryDir /run/passenger-instreg" >> "$passenger_conf" conf_dir=/var/www/canvas/config -sed -i "/worker_max_memory_usage:/ s|:.*|:1073741824|" "$conf_dir/security.yml" +sed -i "/worker_max_memory_usage:/ s|:.*|:1073741824|" "$conf_dir/delayed_jobs.yml" sed -i "/lti_iss:/ S|:.*|: \"https://www.example.com\"" "$conf_dir/security.yml" py3clean /