Merge branch 'main' into agentV5

Author: SQLDBAWithABeard

CONTRIBUTING.md

@@ -1,6 +1,5 @@
 # Contributing
 
-## TODO
 ## Welcome
 
 Before we go any further, thanks for being here. Thanks for using dbachecks and especially thanks 
@@ -34,7 +33,7 @@ In order to use the devcontainer there are a few things you need to get started.
 - [Docker](https://www.docker.com/get-started)
 - [git](https://git-scm.com/downloads)
 - [VSCode](https://code.visualstudio.com/download)
-- [`Remote Development` Extension for VSCode](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack)
+- [Remote Development Extension for VSCode](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack)
 
 ### Setup
 
@@ -61,9 +60,81 @@ so that could take a hot second depending on your internet speeds.
 1. The first time you do this it may take a little, and you'll need an internet connection, as it'll download the container images used in our demos
 
 ### Develop & Build
-TODO: sampler instructions - similar to /workspace/developing/Howto.md
 
-### Rebuild
+We are using the [Sampler](https://github.com/gaelcolas/Sampler) Powershell Module to structure our module.
+This makes it easier to develop and test the module locally.
+
+The workflow for using this and developing the code - for example to add a new Database level check you could follow
+this guide.
+
+1. Download the repo locally and create a new branch to develop on
+    ```PowerShell
+    git checkout -b newStuff # give it a proper name!
+    ```
+
+1. Develop in the source repository, to add a check you need to add the following code:
+  - add check code to `source/checks/DatabaseV5.Tests.ps1`
+  - add required configurations to `source/internal/configurations/configuration.ps1`
+    - `skip.database.checkName`
+    - `policy.database.checkNameExcludeDb`
+  - add required properties to object info to `source/internal/functions/Get-AllDatabaseInfo.ps1`
+
+1. Build the module
+    ```PowerShell
+    ./build.ps1 -Tasks build
+    ```
+
+1. Sampler automatically adds the new version to your path you can prove that with the following code:
+    ```PowerShell
+    get-module dbachecks -ListAvailable | Select-Object Name, ModuleBase
+    ```
+
+1. Import new version of the module
+    ```PowerShell
+    Import-Module dbachecks -force
+    ```
+
+1. Test out the new code
+    ```PowerShell
+    # save the password to make for easy connections
+    $password = ConvertTo-SecureString "dbatools.IO" -AsPlainText -Force
+    $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList "sqladmin", $password
+
+    $show = 'All'
+    $checks = 'RecoveryModel' # <-- change this to your new check name 
+
+    $sqlinstances = 'localhost,7401', 'localhost,7402', 'localhost,7403'
+    #$sqlinstances = 'dbachecks1', 'dbachecks2', 'dbachecks3' # need client aliases for this to work New-DbaClientAlias
+
+    # Run v5 checks
+    $v5code = Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check $Checks -legacy $false -Show $show -PassThru -Verbose
+    ```
+
+1. If you are working on the v4 --> v5 upgrade you can also confirm your v5 test results match v4 with the following
+    ```PowerShell
+    # save the password to make for easy connections
+    $password = ConvertTo-SecureString "dbatools.IO" -AsPlainText -Force
+    $cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList "sqladmin", $password
+
+    $show = 'All'
+    $checks = 'RecoveryModel' # <-- change this to your new check name 
+
+    $sqlinstances = 'localhost,7401', 'localhost,7402', 'localhost,7403'
+    #$sqlinstances = 'dbachecks1', 'dbachecks2', 'dbachecks3' # need client aliases for this to work New-DbaClientAlias
+
+    # Check results of the tests - are we testing the same things with the same results for v4 & v5
+    Invoke-PerfAndValidateCheck -SQLInstances $sqlinstances -Checks $Checks
+    # Include the specific details for the perf testing
+    Invoke-PerfAndValidateCheck -SQLInstances $sqlinstances -Checks $Checks -PerfDetail
+    # Include the test results - this helps troubleshooting if your tests aren't the same
+    Invoke-PerfAndValidateCheck -SQLInstances $sqlinstances -Checks $Checks -showTestResults
+    ```
+
+1. Once you are happy with your code, push your branch to GitHub and create a PR against the dbachecks repo.
+
+1. Thanks!
+
+### Rebuild your devcontainer
 
 The only way to properly rebuild to ensure that all volumes etc are removed is to open up a console
 or PowerShell window outside of the devcontainer and run the following:

containers/JessAndBeard.psm1

@@ -2304,8 +2304,9 @@ New Code runs in {2} % of the time
   Write-PSFMessage -Message $savingMessage -Level Output
 
   ##validate we got the right answers too
-
-  If (Compare-Object $v5code.Configuration.Filter.Tag.Value $v4code.TagFilter) {
+[System.Collections.ArrayList]$v5CodeTags = $v5code.Configuration.Filter.Tag.Value
+$v5CodeTags.Remove('FailedConnections')
+  If (Compare-Object $v5CodeTags $v4code.TagFilter) {
       $Message = "
 Uh-Oh - The Tag filters between v4 and v5 are not the same somehow.
 For v4 We returned
@@ -2427,7 +2428,12 @@ For v4 We Passed
 and
 for v4 we Passed
 {1} tests
-{2}" -f $v4code.PassedCount, $v5Passed, $messageAppend
+{2}
+v4 TestNames
+{3}
+v5 TestNames
+{4}
+" -f $v4code.PassedCount, $v5Passed, $messageAppend, ($v4code.TestResult.Where{$_.Result -eq 'Passed'}.Name |Out-String),($v5code.Passed.ExpandedName |Out-String)
       Write-PSFMessage -Message $Message -Level Warning
   } else {
       $message = "
@@ -2444,7 +2450,11 @@ For v4 We Failed
 and
 for v4 we Failed
 {1} tests
-" -f $v4code.FailedCount, $v5failed
+v4 TestNames
+{2}
+v5 TestNames
+{3}
+" -f $v4code.FailedCount, $v5failed,($v4code.TestResult.Where{$_.Result -eq 'Failed'}.Name |Out-String),($v5code.Failed.ExpandedName |Out-String)
       Write-PSFMessage -Message $Message -Level Warning
   } else {
       $message = "
@@ -2459,9 +2469,13 @@ Uh-Oh - The total tests Skipped between v4 and v5 are not the same somehow.
 For v4 We Skipped
 {0} tests
 and
-for v4 we Skipped
-{1} tests
-{2}" -f $v4code.SkippedCount, $v5skipped, $messageAppend
+For v5 we Skipped
+{1}
+v4 TestNames
+{3}
+v5 TestNames
+{4}
+{2}" -f $v4code.SkippedCount, $v5skipped, $messageAppend,($v4code.TestResult.Where{$_.Result -eq 'Skipped'}.Name |Out-String),($v5code.Skipped.ExpandedName |Out-String)
       Write-PSFMessage -Message $Message -Level Warning
   } else {
       $message = "

containers/base/profile.ps1

@@ -59,8 +59,8 @@ function Load-Profile {
     )
     $global:__currentTheme = (Get-Random -InputObject $themes)
     function global:Get-CurrentPoshTheme { $__currentTheme }
-    Set-PoshPrompt -Theme $__currentTheme
-
+    # Set-PoshPrompt -Theme $__currentTheme
+    Set-PoshPrompt -Theme 'chips'
     if ($psstyle) {
         $psstyle.FileInfo.Directory = $psstyle.FileInfo.Executable = $psstyle.FileInfo.SymbolicLink = ""
         $PSStyle.FileInfo.Extension.Clear()

developing/Oslo Demo.ps1

@@ -0,0 +1,109 @@
+# Oslo Demo
+
+./build.ps1 -tasks build
+
+#region setup
+$containers = $SQLInstances = $dbachecks1, $dbachecks2, $dbachecks3 = 'dbachecks1', 'dbachecks2', 'dbachecks3'
+$password = ConvertTo-SecureString "dbatools.IO" -AsPlainText -Force
+$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList "sqladmin", $password
+$show = 'All'
+#endregion
+
+#region What do we have?
+
+Get-DbaDatabase -SqlInstance $Sqlinstances -SqlCredential $cred -ExcludeSystem | Select-Object Sqlinstance, DatabaseName, Status
+
+Get-DbaAgentJob -SqlInstance $Sqlinstances -SqlCredential $cred | Select-Object Sqlinstance, Name, Enabled
+#end region
+
+Get-DbaLastBackup -SqlInstance $Sqlinstances -SqlCredential $cred | Select-Object Sqlinstance, Database, LastFullBackup | Format-Table
+
+# lets run a couple of tests
+
+# this one shows that the old existing code will work
+# the legacy switch is set to true by default
+
+Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check InstanceConnection, DatabaseStatus -Show $show
+
+# So lets show the shiny new faster code - legacy switch set to false
+
+Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check InstanceConnection, DatabaseStatus -Show $show -legacy $false
+
+
+# The Authentication check failed but we  would like to pass  - lets  set config
+Set-DbcConfig -Name policy.connection.authscheme -Value SQL
+
+# run again
+
+Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check InstanceConnection, DatabaseStatus -Show $show -legacy $false
+
+# Hmmm, we know that we will never be able to remote onto these containers so let talk about skipping. No Claudio not that sort of skipping!!
+Set-DbcConfig -Name skip.connection.remoting -Value $true
+
+# run again
+
+Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check InstanceConnection, DatabaseStatus -Show $show -legacy $false
+
+# So much quicker !!! OK for one check it will be slower. For two it will probably be about the same but for 3 or more it will be quicker. Much quicket. Exrapolate that to 100 checks and a 1000 instances you can see the difference.
+
+# This is how we know - We perf test our PowerShell code
+# This will take about 80-100 seconds to run so run first then talk!
+
+$Checks = 'ErrorLogCount', 'XpCmdShellDisabled', 'WhoIsActiveInstalled', 'CLREnabled', 'TwoDigitYearCutoff', 'MaxDopInstance', 'ErrorLogCount', 'ModelDbGrowth', 'DefaultBackupCompression', 'SaExist', 'SaDisabled', 'SaRenamed', 'DefaultFilePath', 'AdHocDistributedQueriesEnabled', 'AdHocWorkload', 'DefaultTrace', 'OleAutomationProceduresDisabled', 'CrossDBOwnershipChaining', 'ScanForStartupProceduresDisabled', 'RemoteAccessDisabled', 'SQLMailXPsDisabled', 'DAC', 'OLEAutomation', 'ServerNameMatch', 'OrphanedFile', 'MaxMemory', 'PublicPermission'
+
+Invoke-PerfAndValidateCheck -Checks $Checks
+
+# I want to use the results in a different way
+
+# ok lets run the checks and save the out put to a variable so that we can show you what happens. Notice we need the -PassThru switch
+
+$CheckResults = Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check InstanceConnection, DatabaseStatus -Show $show -legacy $false -PassThru
+
+# this is our base results object
+$CheckResults
+
+# lets convert it to something useful
+
+$SomethingUseful = $CheckResults | Convert-DbcResult
+
+$SomethingUseful
+$SomethingUseful | Format-Table
+
+$SomethingUseful | Select-Object -First 1
+
+# Label huh - what is that?
+# label these results so that they can be filtered later
+
+$Coffee = $CheckResults | Convert-DbcResult -Label 'CoffeeFilter'
+
+$Coffee | Select-Object -First 1
+
+# Now we can set those to a file if we want
+
+$CheckResults | Convert-DbcResult -Label 'CoffeeFilter' | Set-DbcFile -FileType Json -FilePath . -FileName oslo -Verbose
+$CheckResults | Convert-DbcResult -Label 'Whiskey' | Set-DbcFile -FileType Json -FilePath . -FileName oslo -Append
+
+code ./oslo.json
+
+# or put them into a database table
+
+$CheckResults | Convert-DbcResult -Label 'CoffeeFilter' | Write-DbcTable -SqlInstance dbachecks1 -SqlCredential $cred -Database tempdb -Verbose
+
+Invoke-DbaQuery -SqlInstance dbachecks1 -SqlCredential $cred -Database tempdb -Query 'SELECT * FROM CheckResults'
+
+# YOU CANT DO THIS FROM HERE - Open Windows terminal on the host and run
+
+Start-DbcPowerBi -FromDatabase
+
+# then use localhost,7401 tempdb and u:sqladmin p:dbatools.IO
+
+# question turn off a container adn talk about hte fails?
+
+
+## made some funky results for the Power Bi
+
+$CheckResults = Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check Instance, Database -Show $show -legacy $false -PassThru
+
+$CheckResults | Convert-DbcResult -Label 'DatabaseInstance' | Write-DbcTable -SqlInstance dbachecks1 -SqlCredential $cred -Database tempdb -Verbose
+
+$CheckResults = Invoke-DbcCheck -SqlInstance $Sqlinstances -SqlCredential $cred -Check compatibilitylevel -Show $show -legacy $false -PassThru

developing/Robs-Instance.ps1

@@ -1,4 +1,6 @@
-$Checks = 'ErrorLogCount', 'XESessionExists', 'XESessionStopped', 'XpCmdShellDisabled', 'WhoIsActiveInstalled', 'CLREnabled', 'TwoDigitYearCutoff', 'MaxDopInstance', 'ErrorLogCount', 'ModelDbGrowth', 'DefaultBackupCompression', 'SaExist', 'SaDisabled', 'SaRenamed', 'DefaultFilePath', 'AdHocDistributedQueriesEnabled', 'AdHocWorkload', 'DefaultTrace', 'OleAutomationProceduresDisabled', 'CrossDBOwnershipChaining', 'ScanForStartupProceduresDisabled', 'RemoteAccessDisabled', 'SQLMailXPsDisabled', 'DAC', 'OLEAutomation', 'ServerNameMatch', 'OrphanedFile', 'MaxMemory', 'NetworkLatency'
+./build.ps1 -Tasks build
+
+$Checks = 'ErrorLogCount', 'XESessionExists', 'XESessionStopped', 'XpCmdShellDisabled', 'WhoIsActiveInstalled', 'CLREnabled', 'TwoDigitYearCutoff', 'MaxDopInstance', 'ErrorLogCount', 'ModelDbGrowth', 'DefaultBackupCompression', 'SaExist', 'SaDisabled', 'SaRenamed', 'DefaultFilePath', 'AdHocDistributedQueriesEnabled', 'AdHocWorkload', 'DefaultTrace', 'OleAutomationProceduresDisabled', 'CrossDBOwnershipChaining', 'ScanForStartupProceduresDisabled', 'RemoteAccessDisabled', 'SQLMailXPsDisabled', 'DAC', 'OLEAutomation', 'ServerNameMatch', 'OrphanedFile', 'MaxMemory', 'NetworkLatency', 'PublicPermission'
 
 $Checks = 'XESessionRunningAllowed', 'XESessionRunning', 'XESessionRunningAllowed', 'XESessionExists', 'XESessionStopped', 'XpCmdShellDisabled'
 $Checks = 'TraceFlagsNotExpected', 'TraceFlagsExpected'
@@ -18,6 +20,9 @@ $Checks = 'SuspectPageLimit'
 $Checks = 'SupportedBuild'
 $Checks = 'LoginMustChange'
 $Checks = 'LoginAuditSuccessful', 'LoginAuditFailed'
+Set-DbcConfig -Name skip.security.PublicPermission -Value $false
+$Checks = 'PublicRolePermission'
+$Checks = 'PUblicPermission'
 
 Invoke-PerfAndValidateCheck -Checks $Checks
 Invoke-PerfAndValidateCheck -Checks $Checks -PerfDetail

source/checks/Databasev5.Tests.ps1

@@ -58,17 +58,17 @@ Describe "Suspect Page" -Tag SuspectPage, High , Database -ForEach $InstancesToT
 }
 
 Describe "Database Collation" -Tag DatabaseCollation, High, Database -ForEach $InstancesToTest {
+    #TODO: Should we have a skip option for each IT block?
     $skip = ($__dbcconfig | Where-Object { $_.Name -eq 'skip.database.databasecollation' }).Value
     Context "Testing database collation on <_.Name>" {
         It "Database <_.Name> collation <_.Collation> should match server collation <_.ServerCollation> on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database } else { $psitem.ConfigValues.wrongcollation -notcontains $PsItem.Name } } {
             $psitem.ServerCollation | Should -Be $psitem.Collation -Because "You will get collation conflict errors in tempdb"
         }
 
         # wrong collation set
-        It "Database <_.Name> collation <_.Collation> should not match server collation <_.ServerCollation> on <_.SqlInstance>" -ForEach $psitem.Databases.Where{ $_.Name -in $psitem.ConfigValues.wrongcollation } {
+        It "Database <_.Name> collation <_.Collation> should not match server collation <_.ServerCollation> on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ $_.Name -in $psitem.ConfigValues.wrongcollation } {
             $psitem.ServerCollation | Should -Not -Be $psitem.Collation -Because "You have defined the database to have another collation then the server. You will get collation conflict errors in tempdb"
         }
-
     }
 }
 
@@ -216,7 +216,7 @@ Describe "Query Store Disabled" -Tag QueryStoreDisabled, Medium, Database -ForEa
 Describe "Compatibility Level" -Tag CompatibilityLevel, High, Database -ForEach $InstancesToTest {
     $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.database.compatibilitylevel').Value
 
-    Context "Compatibility level matches server compatibility level" {
+    Context "Compatibility level matches server compatibility level on <_.Name>" {
         It "Database <_.Name> has the expected compatibility level on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database } else { $psitem.ConfigValues.compatexclude -notcontains $psitem.Name } } {
             $psitem.CompatibilityLevel | Should -Be $psitem.ServerLevel -Because "it means you are on the appropriate compatibility level for your SQL Server version to use all available features."
         }
@@ -226,7 +226,7 @@ Describe "Compatibility Level" -Tag CompatibilityLevel, High, Database -ForEach
 Describe "Guest User" -Tag GuestUserConnect, Security, CIS, Medium, Database -ForEach $InstancesToTest {
     $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.security.guestuserconnect').Value
 
-    Context "Testing Guest user has CONNECT permission" {
+    Context "Testing Guest user has CONNECT permission on <_.Name>" {
         It "Database Guest user should return no CONNECT permissions in <_.Name> on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database } else { $psitem.ConfigValues.guestuserexclude -notcontains $psitem.Name } } {
             $psitem.GuestUserConnect | Should -BeFalse -Because "we don't want the guest user to have connect access to our database."
         }
@@ -236,9 +236,41 @@ Describe "Guest User" -Tag GuestUserConnect, Security, CIS, Medium, Database -Fo
 Describe "Recovery Model" -Tag RecoveryModel, DISA, Medium, Database -ForEach $InstancesToTest {
     $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.database.recoverymodel').Value
 
-    Context "Testing Recovery Model" {
+    Context "Testing Recovery Model on <_.Name>" {
         It "Database <_.Name> should be set to <_.ConfigValues.recoverymodeltype> on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database } else { $psitem.ConfigValues.recoverymodelexclude -notcontains $psitem.Name } } {
             $psitem.RecoveryModel | Should -Be $psitem.ConfigValues.recoverymodeltype -Because "You expect this recovery model."
         }
     }
 }
+
+Describe "PseudoSimple Recovery Model" -Tag PseudoSimple, Medium, Database -ForEach $InstancesToTest {
+    $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.database.pseudosimple').Value
+
+    Context "Testing database is not in PseudoSimple recovery model on <_.Name>" {
+        It "Database <_.Name> has PseudoSimple recovery model equal false on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database -and $_.RecoveryModel -eq 'Full' } else { $psitem.ConfigValues.pseudosimpleexclude -notcontains $psitem.Name -and $_.RecoveryModel -eq 'Full' } } {
+            $psitem.PseudoSimple | Should -BeFalse -Because "PseudoSimple means that a FULL backup has not been taken and the database is still effectively in SIMPLE mode"
+        }
+    }
+}
+
+Describe "Contained Database Auto Close" -Tag ContainedDBAutoClose, CIS, Database -ForEach $InstancesToTest {
+    $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.security.containedbautoclose').Value
+
+    Context "Testing contained database auto close option on <_.Name>" {
+        It "Database <_.Name> should have auto close set to false on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database -and $_.ContainmentType -ne "NONE" } else { $psitem.ConfigValues.contdbautocloseexclude -notcontains $psitem.Name -and $_.ContainmentType -ne "NONE" } } {
+            $psitem.ContainedDbAutoClose | Should -BeFalse -Because "Contained Databases should have auto close set to false for CIS compliance."
+        }
+    }
+}
+
+Describe "Contained Database SQL Authenticated Users" -Tag ContainedDBSQLAuth, CIS, Database -ForEach $InstancesToTest {
+    $Skip = ($__dbcconfig | Where-Object Name -EQ 'skip.security.ContainedDBSQLAuth').Value
+
+    #if ($version -lt 13 ) { $skip = $true }
+
+    Context "Testing contained database to see if sql authenticated users exist on <_.Name>" {
+        It "Database <_.Name> should have no sql authenticated users on <_.SqlInstance>" -Skip:$skip -ForEach $psitem.Databases.Where{ if ($Database) { $_.Name -in $Database -and $_.ContainmentType -ne "NONE" } else { $psitem.ConfigValues.contdbsqlauthexclude -notcontains $psitem.Name -and $_.ContainmentType -ne "NONE" } } {
+            $psitem.ContainedDbSqlAuthUsers | Should -Be 0 -Because "We expect there to be no sql authenticated users in contained database."
+        }
+    }
+}