From 8e6b2c86f2f4e54bd3bbd61e8c4a2cc4c81731bd Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Mon, 10 Feb 2025 22:42:56 -0800 Subject: [PATCH 1/3] Add puppetcore support for Windows install tasks Now possible to run the install task specifying puppetcore collection: ``` /opt/puppetlabs/bolt/bin/bolt task run puppet_agent::install \ collection=puppetcore8 \ version=8.11.0 \ username=forge-key \ password=${PUPPET_FORGE_TOKEN} \ --targets 'winrm://HOST' \ --user Administrator \ --password ... ``` If the `windows_source` class parameter is explicitly given, then the task will use that. Also add additional logging as to where we are downloading the MSI from and the exception message if downloading fails. --- tasks/install_powershell.json | 8 +++++++ tasks/install_powershell.ps1 | 39 ++++++++++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/tasks/install_powershell.json b/tasks/install_powershell.json index 868a95c6..8d64afb0 100644 --- a/tasks/install_powershell.json +++ b/tasks/install_powershell.json @@ -42,6 +42,14 @@ "description": "The number of retries in case of network connectivity failures", "type": "Optional[Integer]", "default": 5 + }, + "username": { + "description": "The username to use when downloading from a source location requiring authentication", + "type": "Optional[String]" + }, + "password": { + "description": "The password to use when downloading from a source location requiring authentication", + "type": "Optional[String]" } }, "supports_noop": true diff --git a/tasks/install_powershell.ps1 b/tasks/install_powershell.ps1 index 679baa07..6aaf26ef 100644 --- a/tasks/install_powershell.ps1 +++ b/tasks/install_powershell.ps1 @@ -7,11 +7,21 @@ Param( [String]$install_options = 'REINSTALLMODE="amus"', [Bool]$stop_service = $False, [Int]$retry = 5, - [Bool]$_noop = $False + [Bool]$_noop = $False, + [String]$username = 'forge-key', + [String]$password ) # If an error is encountered, the script will stop instead of the default of "Continue" $ErrorActionPreference = "Stop" +try { + $os_version = (Get-WmiObject Win32_OperatingSystem).Version +} +catch [System.Management.Automation.CommandNotFoundException] { + $os_version = (Get-CimInstance -ClassName win32_OperatingSystem).Version +} +$major_os_version = ($os_version -split '\.')[0] + try { if ((Get-WmiObject Win32_OperatingSystem).OSArchitecture -match '^32') { $arch = "x86" @@ -27,9 +37,19 @@ catch [System.Management.Automation.CommandNotFoundException] { } } +$fips = 'false' +try { + if ((Get-ItemPropertyValue -Path 'HKLM:\System\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy' -Name Enabled) -ne 0) { + $fips = 'true' + } +} +catch { + Write-Output "Failed to lookup FIPS mode, assuming it is disabled" +} + function Test-PuppetInstalled { $rootPath = 'HKLM:\SOFTWARE\Puppet Labs\Puppet' - try { + try { if (Get-ItemProperty -Path $rootPath) { RETURN $true } } catch { @@ -98,12 +118,16 @@ if (Test-RunningServices) { # Change windows_source only if the collection is a nightly build, and the source was not explicitly specified. if (($collection -like '*nightly*') -And -Not ($PSBoundParameters.ContainsKey('windows_source'))) { $windows_source = 'https://nightlies.puppet.com/downloads' +} elseif (($collection -like '*puppetcore*') -And -Not ($PSBoundParameters.ContainsKey('windows_source'))) { + $windows_source = 'https://artifacts-puppetcore.puppet.com/v1/download' } if ($absolute_source) { $msi_source = "$absolute_source" } -else { +elseif ($collection -like '*puppetcore*') { + $msi_source = "${windows_source}?version=${version}&os_name=windows&os_version=${major_os_version}&os_arch=${arch}&fips=${fips}" +} else { $msi_source = "$windows_source/windows/${collection}/${msi_name}" } @@ -125,15 +149,19 @@ function Set-Tls12 { } function DownloadPuppet { - Write-Output "Downloading the Puppet Agent installer on $env:COMPUTERNAME..." + Write-Output "Downloading the Puppet Agent installer on $env:COMPUTERNAME from ${msi_source}" Set-Tls12 $webclient = New-Object system.net.webclient - + if ($password) { + $credentials = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes("${username}:${password}")) + $webclient.Headers.Add("Authorization", "Basic ${credentials}") + } try { $webclient.DownloadFile($msi_source,$msi_dest) } catch [System.Net.WebException] { + Write-Host "Download exception: $($_.Exception.Message)" For ($attempt_number = 1; $attempt_number -le $retry; $attempt_number++) { try { Write-Output "Retrying... [$attempt_number/$retry]" @@ -141,6 +169,7 @@ function DownloadPuppet { break } catch [System.Net.WebException] { + Write-Host "Download exception: $($_.Exception.Message)" if($attempt_number -eq $retry) { # If we can't find the msi, then we may not be configured correctly if($_.Exception.Response.StatusCode -eq [system.net.httpstatuscode]::NotFound) { From 33cd0f6f61e7f0dcb8d27b35d1d880622f272104 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Tue, 18 Mar 2025 12:50:35 -0700 Subject: [PATCH 2/3] Upgrade puppetcore* msi from artifacts-puppetcore.puppet.com When using the puppetcore collection on Windows, if we detect the installed version does not match, then upgrade the MSI. Due to a puppet bug, we cannot pass credentials in the `source` parameter. And `curl.exe` is not present in our puppet-agent packages. So use powershell to download. Co-authored-by: Kevin <114269618+klab-systems@users.noreply.github.com> --- REFERENCE.md | 23 +++++++++++ manifests/osfamily/windows.pp | 8 +++- manifests/prepare/package.pp | 65 ++++++++++++++++++++++++------- templates/download_puppet.ps1.epp | 19 +++++++++ 4 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 templates/download_puppet.ps1.epp diff --git a/REFERENCE.md b/REFERENCE.md index 8f2a2319..fa6cb61e 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -624,6 +624,7 @@ working with a remote https repository. The following parameters are available in the `puppet_agent::prepare::package` class: * [`source`](#-puppet_agent--prepare--package--source) +* [`package_file_name`](#-puppet_agent--prepare--package--package_file_name) ##### `source` @@ -632,6 +633,16 @@ Data type: `Variant[String, Array]` The source file for the puppet-agent package. Can use any of the data types and protocols that the File resource's source attribute can. +##### `package_file_name` + +Data type: `Optional[String]` + +The destination file name for the puppet-agent package. If no destination +is given, then the basename component of the source will be used as the +destination filename. + +Default value: `undef` + ### `puppet_agent::prepare::puppet_config` Private class called from puppet_agent::prepare class. @@ -993,6 +1004,18 @@ Data type: `Optional[Integer]` The number of retries in case of network connectivity failures +##### `username` + +Data type: `Optional[String]` + +The username to use when downloading from a source location requiring authentication + +##### `password` + +Data type: `Optional[String]` + +The password to use when downloading from a source location requiring authentication + ### `install_shell` Install the Puppet agent package diff --git a/manifests/osfamily/windows.pp b/manifests/osfamily/windows.pp index 8114ddeb..a97bbcd7 100644 --- a/manifests/osfamily/windows.pp +++ b/manifests/osfamily/windows.pp @@ -2,6 +2,8 @@ class puppet_agent::osfamily::windows { assert_private() + $destination_name = undef + if $puppet_agent::absolute_source { $source = $puppet_agent::absolute_source } elsif $puppet_agent::source { @@ -23,13 +25,17 @@ } else { if $puppet_agent::collection == 'PC1' { $source = "${puppet_agent::windows_source}/windows/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-${puppet_agent::arch}.msi" + } elsif $puppet_agent::collection =~ /core/ { + $source = 'https://artifacts-puppetcore.puppet.com/v1/download' + $destination_name = "${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-${puppet_agent::arch}.msi" } else { $source = "${puppet_agent::windows_source}/windows/${puppet_agent::collection}/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-${puppet_agent::arch}.msi" } } class { 'puppet_agent::prepare::package': - source => $source, + source => $source, + destination_name => $destination_name, } contain puppet_agent::prepare::package diff --git a/manifests/prepare/package.pp b/manifests/prepare/package.pp index dee0afa2..6654a12e 100644 --- a/manifests/prepare/package.pp +++ b/manifests/prepare/package.pp @@ -5,8 +5,13 @@ # @param source # The source file for the puppet-agent package. Can use any of the data types # and protocols that the File resource's source attribute can. +# @param destination_name +# The destination file name for the puppet-agent package. If no destination +# is given, then the basename component of the source will be used as the +# destination name. class puppet_agent::prepare::package ( Variant[String, Array] $source, + Optional[String] $destination_name = undef ) { assert_private() @@ -14,12 +19,17 @@ ensure => directory, } - # In order for the 'basename' function to work correctly we need to change - # any \s to /s (even for windows UNC paths) so that it will correctly pull off - # the filename. Since this operation is only grabbing the base filename and not - # any part of the path this should be safe, since the source will simply remain - # what it was before and we can still pull off the filename. - $package_file_name = basename(regsubst($source, "\\\\", '/', 'G')) + if $destination_name { + $package_file_name = $destination_name + } else { + # In order for the 'basename' function to work correctly we need to change + # any \s to /s (even for windows UNC paths) so that it will correctly pull off + # the filename. Since this operation is only grabbing the base filename and not + # any part of the path this should be safe, since the source will simply remain + # what it was before and we can still pull off the filename. + $package_file_name = basename(regsubst($source, "\\\\", '/', 'G')) + } + if $facts['os']['family'] =~ /windows/ { $local_package_file_path = windows_native_path("${puppet_agent::params::local_packages_dir}/${package_file_name}") $mode = undef @@ -28,12 +38,41 @@ $mode = '0644' } - file { $local_package_file_path: - ensure => file, - owner => $puppet_agent::params::user, - group => $puppet_agent::params::group, - mode => $mode, - source => $source, - require => File[$puppet_agent::params::local_packages_dir], + # REMIND: redhat/suse with absolute_source + # REMIND: debian with absolute_source + # REMIND: solaris 10 + # REMIND: solaris 11 with manage_repo + # REMIND: aix + # REMIND: darwin + # REMIND: suse 11 and PE + if $puppet_agent::collection =~ /core/ and $facts['os']['family'] =~ /windows/ { + $download_username = getvar('puppet_agent::username', 'forge-key') + $download_password = unwrap(getvar('puppet_agent::password')) + + $_download_puppet = windows_native_path("${facts['env_temp_variable']}/download_puppet.ps1") + file { $_download_puppet: + ensure => file, + content => Sensitive(epp('puppet_agent/download_puppet.ps1.epp')), + } + + exec { 'Download Puppet Agent': + command => "${facts['os']['windows']['system32']}\\WindowsPowerShell\\v1.0\\powershell.exe \ + -ExecutionPolicy Bypass \ + -NoProfile \ + -NoLogo \ + -NonInteractive \ + ${_download_puppet}", + creates => $local_package_file_path, + provider => powershell, + } + } else { + file { $local_package_file_path: + ensure => file, + owner => $puppet_agent::params::user, + group => $puppet_agent::params::group, + mode => $mode, + source => $source, + require => File[$puppet_agent::params::local_packages_dir], + } } } diff --git a/templates/download_puppet.ps1.epp b/templates/download_puppet.ps1.epp new file mode 100644 index 00000000..bf9ff707 --- /dev/null +++ b/templates/download_puppet.ps1.epp @@ -0,0 +1,19 @@ +$body = @{ + "version" = "<%= $puppet_agent::prepare::package_version %>" + "os_name" = "<%= $facts['os']['family'] %>" + "os_version" = "<%= $facts['os']['release']['major'] %>" + "os_arch" = "<%= $facts['os']['architecture'] %>" + "fips" = "<%= $facts['fips_enabled'] %>" +} +$username = "<%= $puppet_agent::prepare::package::download_username %>" +$password = ConvertTo-SecureString "<%= $puppet_agent::prepare::package::download_password %>" -AsPlainText -Force +$credential = New-Object System.Management.Automation.PSCredential($username, $password) +try { + Invoke-WebRequest -Uri "<%= $puppet_agent::prepare::package::source %>" ` + -Body $body ` + -Credential $credential ` + -OutFile "<%= $puppet_agent::prepare::package::local_package_file_path %>" +} catch [System.Net.WebException] { + Write-Host "Network-related error: $($_.Exception.Message)" + exit 1 +} From 191a2ccb85608b2ae1d8263a9771d706ae89d6af Mon Sep 17 00:00:00 2001 From: Christopher Thorn Date: Tue, 1 Apr 2025 14:14:51 -0700 Subject: [PATCH 3/3] (maint) Update MacOS for puppetcore work This updates the download of puppet-agent when puppetcore packages are used. The new 'puppetcore7' and 'puppetcore8' collections when used for MacOS will now download puppetcore packages. Due to a bug in Puppet for now we're going to depend on Curl to download the package. --- manifests/osfamily/darwin.pp | 10 +++++- manifests/prepare/package.pp | 40 +++++++++++++++++++++++- tasks/install_shell.sh | 60 ++++++++++++++++++++++++++++-------- 3 files changed, 95 insertions(+), 15 deletions(-) diff --git a/manifests/osfamily/darwin.pp b/manifests/osfamily/darwin.pp index ab3dfd15..f55023bc 100644 --- a/manifests/osfamily/darwin.pp +++ b/manifests/osfamily/darwin.pp @@ -20,12 +20,20 @@ } else { $source = "puppet:///pe_packages/${pe_server_version}/${facts['platform_tag']}/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${$productversion_major}.dmg" } - } else { + } elsif $puppet_agent::collection and $puppet_agent::collection =~ /core/ { + if $puppet_agent::prepare::package_version =~ /^\d+\.\d+\.\d+\.\d+\.g([a-f0-9]+)+$/ { + $source = "https://artifacts-puppetcore.puppet.com/v1/download?type=native&version=${puppet_agent::prepare::package_version}&os_name=osx&os_version=${productversion_major}&os_arch=${puppet_agent::arch}&dev=true" + } else { + $source = "https://artifacts-puppetcore.puppet.com/v1/download?type=native&version=${puppet_agent::prepare::package_version}&os_name=osx&os_version=${productversion_major}&os_arch=${puppet_agent::arch}" + } + $destination_name = "${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${productversion_major}.dmg" + } else { $source = "${puppet_agent::mac_source}/mac/${puppet_agent::collection}/${productversion_major}/${puppet_agent::arch}/${puppet_agent::package_name}-${puppet_agent::prepare::package_version}-1.osx${$productversion_major}.dmg" } class { 'puppet_agent::prepare::package': source => $source, + destination_name => $destination_name, } contain puppet_agent::prepare::package diff --git a/manifests/prepare/package.pp b/manifests/prepare/package.pp index 6654a12e..38db02c7 100644 --- a/manifests/prepare/package.pp +++ b/manifests/prepare/package.pp @@ -45,7 +45,7 @@ # REMIND: aix # REMIND: darwin # REMIND: suse 11 and PE - if $puppet_agent::collection =~ /core/ and $facts['os']['family'] =~ /windows/ { + if $puppet_agent::collection and $puppet_agent::collection =~ /core/ and $facts['os']['family'] =~ /windows/ { $download_username = getvar('puppet_agent::username', 'forge-key') $download_password = unwrap(getvar('puppet_agent::password')) @@ -65,6 +65,44 @@ creates => $local_package_file_path, provider => powershell, } + } elsif $puppet_agent::collection and $puppet_agent::collection =~ /core/ and $facts['os']['family'] =~ /Darwin/ { + $download_username = getvar('puppet_agent::username', 'forge-key') + $download_password = unwrap(getvar('puppet_agent::password')) + + $response_file = "${local_package_file_path}.response" + $netrc_file = "${facts['env_temp_variable']}/.netrc" + file { $netrc_file: + ensure => file, + content => "machine artifacts-puppetcore.puppet.com\nlogin ${download_username}\npassword ${download_password}\n", + mode => '0600', + } + + $curl_command = "curl -1 -sL --netrc-file '${netrc_file}' -w '%{http_code}' -o '${local_package_file_path}' '${source}' > '${response_file}'" + exec { 'Download Puppet Agent for Darwin': + command => $curl_command, + creates => $local_package_file_path, + path => ['/usr/bin', '/usr/sbin', '/bin', '/sbin'], + } + + exec { 'Remove .netrc file': + command => "rm -f '${netrc_file}'", + path => ['/usr/bin', '/bin'], + onlyif => "test -f '${netrc_file}'", + require => Exec['Download Puppet Agent for Darwin'], + } + # + # TODO: This is a temporary workaround to get the HTTP response code from the curl command. + # For now just outputting the response is good enough. + # We need to find a way to interspect this value and fail the catalog if the response + # code is not 200, and then logging the output wont be as important. + # + exec { 'Read HTTP Response Code': + command => "cat '${response_file}'", + path => ['/usr/bin', '/bin'], + onlyif => "test -f '${response_file}'", + logoutput => true, + require => Exec['Download Puppet Agent for Darwin'], + } } else { file { $local_package_file_path: ensure => file, diff --git a/tasks/install_shell.sh b/tasks/install_shell.sh index 504d7409..ef46c7f7 100644 --- a/tasks/install_shell.sh +++ b/tasks/install_shell.sh @@ -162,10 +162,18 @@ fi if [ -n "$PT_mac_source" ]; then mac_source=$PT_mac_source else - if [ "$nightly" = true ]; then - mac_source='http://nightlies.puppet.com/downloads' - else - mac_source='http://downloads.puppet.com' + if [[ "$PT_collection" =~ core ]]; then + if [ -z "$password" ]; then + echo "A password parameter is required to install with puppetcore" + exit 1 + fi + mac_source='https://artifacts-puppetcore.puppet.com/v1/download' + else + if [ "$nightly" = true ]; then + mac_source='http://nightlies.puppet.com/downloads' + else + mac_source='http://downloads.puppet.com' + fi fi fi @@ -421,7 +429,11 @@ do_wget() { # do_curl URL FILENAME do_curl() { info "Trying curl..." - run_cmd "curl -1 -sL -D $tmp_stderr '$1' > '$2'" + if [[ -n "$3" && -n "$4" ]]; then + run_cmd "curl -1 -sL -u '$3:$4' -D $tmp_stderr '$1' > '$2'" + else + run_cmd "curl -1 -sL -D $tmp_stderr '$1' > '$2'" + fi rc=$? # check for 404 @@ -431,6 +443,12 @@ do_curl() { unable_to_retrieve_package fi + grep "HTTP/2 401" $tmp_stderr 2>&1 >/dev/null + if test $? -eq 0; then + critical "ERROR 401: Unauthorized access" + unable_to_retrieve_package + fi + # check for bad return status or empty output if test $rc -ne 0 || test ! -s "$2"; then capture_tmp_stderr "curl" @@ -557,7 +575,11 @@ do_download() { fi if exists curl; then - do_curl $1 $2 && return 0 + if [[ "$collection" =~ core ]]; then + do_curl $1 $2 "$username" "$password" && return 0 + else + do_curl $1 $2 && return 0 + fi fi if exists fetch; then @@ -810,19 +832,31 @@ case $platform in download_url="${apt_source}/${filename}" ;; "mac_os_x") - info "Mac platform! Lets get you a DMG..." - filetype="dmg" + arch="x86_64" + if [[ $(uname -p) == "arm" ]]; then + arch="arm64" + fi if test "$version" = "latest"; then filename="puppet-agent-latest.dmg" else filename="puppet-agent-${version}-1.osx${platform_version}.dmg" fi - - arch="x86_64" - if [[ $(uname -p) == "arm" ]]; then - arch="arm64" + info "Mac platform! Lets get you a DMG...!!" + if [[ "$collection" =~ core ]]; then + if [ -z "$password" ]; then + echo "A password parameter is required to install" + exit 1 + fi + if [[ "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)\.g([a-f0-9]+)$ ]]; then + download_url="${mac_source}/?version=${version}&os_name=osx&os_version=${platform_version}&os_arch=${arch}&fips=false&dev=true" + else + download_url="${mac_source}/?version=${version}&os_name=osx&os_version=${platform_version}&os_arch=${arch}&fips=false" + fi + else + download_url="${mac_source}/mac/${collection}/${platform_version}/${arch}/${filename}" fi - download_url="${mac_source}/mac/${collection}/${platform_version}/${arch}/${filename}" + filetype="dmg" + ;; *) critical "Sorry $platform is not supported yet!"