Merge pull request #56 from sqlcollaborative/Xplat-tests

Full MySQL coverage: tests and lots of fixes

Author: nvarscar

appveyor.yml

@@ -3,9 +3,9 @@
 build: false
 version: 0.1.{build}
 
-#images 
+#images
 
-image: 
+image:
   - Ubuntu1804
   - Visual Studio 2017
 
@@ -15,38 +15,31 @@ environment:
   version: 0.1.$(appveyor_build_number)
 
   matrix:
-  - scenario: ALL
+  - scenario: all
+#  - scenario: mysql
 
 
 for:
--
-  matrix:
-    only:
-      - image: Visual Studio 2017
-  environment:
-      mssql_instance: localhost\SQL2017
-  services:
-    - mssql2017
+  - matrix:
+      only:
+        - image: Visual Studio 2017
+          #scenario: all
+    environment:
+        mssql_instance: localhost\SQL2017
+    services:
+      - mssql2017
+      - mysql
+
+  - matrix:
+      only:
+        - image: Ubuntu1804
+          #scenario: default
+    environment:
+        mssql_instance: localhost
+    services:
+      - mssql
+      - mysql
 
-- matrix:
-    only:
-      - image: Ubuntu1804
-  environment:
-      mssql_instance: localhost
-      mssql_login: sa
-      mssql_password: Password12!
-  services:
-    - mssql
-  # before_test:
-  #   # run preparation scripts
-  #   - ps: ./tests/appveyor.prep.ps1
-  # test_script:
-  #   # Test with native PS version
-  #   - ps: ./tests/appveyor.pester.ps1
-  #   # Collecting results
-  #   - ps: ./tests/appveyor.pester.ps1 -Finalize
-  # after_test:
-  #   - ps: ./tests/appveyor.post.ps1
 
 before_test:
   # run preparation scripts
@@ -57,12 +50,9 @@ test_script:
   # Collecting results
   - ps: .\tests\appveyor.pester.ps1 -Finalize
 after_test:
-  - ps: .\tests\appveyor.post.ps1  
+  - ps: .\tests\appveyor.post.ps1
+
 
-# #Configure services
-# services:
-#   - mssql2017
-  
 # Set alternative clone folder
 #clone_folder: c:\github\dbops
 

bin/deploy.ps1

@@ -26,9 +26,9 @@ Param (
     [switch]$CreateDatabase,
     [AllowNull()]
     [string]$ConnectionString,
-    [ValidateSet('SQLServer', 'Oracle')]
-    [Alias('Type', 'ServerType')]
-    [string]$ConnectionType = 'SQLServer'
+    [ValidateSet('SqlServer', 'Oracle', 'MySQL', 'PostgreSQL')]
+    [Alias('ConnectionType', 'ServerType')]
+    [string]$Type = 'SQLServer'
 )
 
 #Import module
@@ -39,22 +39,24 @@ If (-not (Get-Module dbops)) {
 
 $config = Get-DBOConfig -Path "$PSScriptRoot\dbops.config.json" -Configuration $Configuration
 
-#Convert custom parameters into a package configuration, excluding variables
+#Merge custom parameters into a configuration
+$newConfig = @{}
 foreach ($key in ($PSBoundParameters.Keys)) {
-    if ($key -in [DBOps.ConfigProperty].GetEnumNames() -and $key -ne 'Variables') {
+    if ($key -in [DBOpsConfig]::EnumProperties()) {
         Write-PSFMessage -Level Debug -Message "Overriding parameter $key with $($PSBoundParameters[$key])"
-        $config.SetValue($key, $PSBoundParameters[$key])
+        $newConfig.$key = $PSBoundParameters[$key]
     }
 }
+$config.Merge($newConfig)
 
 #Prepare deployment function call parameters
 $params = @{
-    PackageFile = "$PSScriptRoot\dbops.package.json"
+    PackageFile   = "$PSScriptRoot\dbops.package.json"
     Configuration = $config
 }
 foreach ($key in ($PSBoundParameters.Keys)) {
     #If any custom properties were specified
-    if ($key -in @('OutputFile', 'Append', 'Variables', 'ConnectionType', 'Build')) {
+    if ($key -in @('OutputFile', 'Append', 'Type', 'Build')) {
         $params += @{ $key = $PSBoundParameters[$key] }
     }
 }

functions/Invoke-DBODeployment.ps1

@@ -155,7 +155,8 @@
 
         Write-PSFMessage -Level Debug -Message "Creating DbUp objects"
         # Create DbUp connection object
-        $connString = Get-ConnectionString -Configuration $config -Type $Type
+        $csBuilder = Get-ConnectionString -Configuration $config -Type $Type -Raw
+        $connString = $csBuilder.ToString()
         $dbUpConnection = Get-ConnectionManager -ConnectionString $connString -Type $Type
 
         # Create DbUpBuilder based on the connection

functions/Invoke-DBOQuery.ps1

@@ -173,20 +173,12 @@ function Invoke-DBOQuery {
         }
 
         #Build connection string
-        #$connString = Get-ConnectionString -Configuration $config -Type $Type
-        #$dbUpConnection = Get-ConnectionManager -ConnectionString $connString -Type $Type
         Write-PSFMessage -Level Debug -Message "Getting the connection object"
-        $dbUpConnection = Get-ConnectionManager -Configuration $config -Type $Type
-        $dbUpSqlParser = Get-SqlParser -Type $Type
-        $status = [DBOpsDeploymentStatus]::new()
-        $dbUpLog = [DBOpsLog]::new($config.Silent, $OutputFile, $Append, $status)
-        $dbUpLog.CallStack = (Get-PSCallStack)[0]
-        if ($null -ne $dbUpConnection.IsScriptOutputLogged -and -Not $config.Silent) {
-            $dbUpConnection.IsScriptOutputLogged = $true
-        }
+        $dbUpConnection = Get-ConnectionManager -ConnectionString $null -Type $Type # only needed for query parsing for now, connection string is pointless
+        $dataConnection = Get-DatabaseConnection -Configuration $config -Type $Type # the 'real' connection
         Write-PSFMessage -Level Verbose -Message "Establishing connection with $Type $($config.SqlInstance)"
         try {
-            $managedConnection = $dbUpConnection.OperationStarting($dbUpLog, $null)
+            $dataConnection.Open()
         }
         catch {
             Stop-PSFFunction -EnableException $true -Message "Failed to connect to the server" -ErrorRecord $_
@@ -216,51 +208,49 @@ function Invoke-DBOQuery {
         foreach ($qText in $queryText) {
             $queryList += Resolve-VariableToken $qText $config.Variables
         }
+
         Write-PSFMessage -Level Debug -Message "Preparing to run $($queryList.Count) queries"
+        $ds = [System.Data.DataSet]::new()
         try {
-            $ds = [System.Data.DataSet]::new()
             $qCount = 0
             foreach ($queryItem in $queryList) {
                 $qCount++
                 if ($PSCmdlet.ShouldProcess("Executing query $qCount", $config.SqlInstance)) {
+                    # split commands using DbUp parser
                     foreach ($splitQuery in $dbUpConnection.SplitScriptIntoCommands($queryItem)) {
-                        $dt = [System.Data.DataTable]::new()
-                        $rows = $dbUpConnection.ExecuteCommandsWithManagedConnection( [Func[Func[Data.IDbCommand], pscustomobject]] {
-                                Param (
-                                    $dbCommandFactory
-                                )
-                                # Compile the parameters as Expression functions
-                                $exp = [System.Linq.Expressions.Expression]
-                                $parameterList = @()
-                                foreach ($key in $Parameter.Keys) {
-                                    $param = $exp::Parameter([string], $key)
-                                    $value = $exp::Constant($Parameter.$key)
-                                    $body = $exp::Convert($value, [System.Object])
-                                    $lambda = $exp::Lambda([Func[string, object]], $body, $param)
-                                    $parameterList += $lambda
-                                }
-                                # Initiating the AdHocSqlRunner
-                                $sqlRunner = [DbUp.Helpers.AdHocSqlRunner]::new($dbCommandFactory, $dbUpSqlParser, $config.Schema)
-                                return $sqlRunner.ExecuteReader($splitQuery, $parameterList)
-                            })
-                        $rowCount = ($rows | Measure-Object).Count
-                        if ($rowCount -gt 0) {
-                            $keys = switch ($rowCount) {
-                                1 { $rows.Keys }
-                                default { $rows[0].Keys }
+                        # create a new command object and define text/parameters
+                        $command = $dataConnection.CreateCommand()
+                        $command.CommandText = $splitQuery
+                        foreach ($key in $Parameter.Keys) {
+                            $null = switch ($Type) {
+                                Oracle { $command.Parameters.Add($key, $Parameter[$key]) }
+                                default { $command.Parameters.AddWithValue($key, $Parameter[$key]) }
                             }
-                            foreach ($column in $keys) {
-                                $null = $dt.Columns.Add($column)
+                        }
+                        # create a reader and define output table columns
+                        $reader = $command.ExecuteReader()
+                        $table = [System.Data.DataTable]::new()
+                        $definition = $reader.GetSchemaTable()
+                        foreach ($column in $definition) {
+                            $name = $column.ColumnName
+                            $datatype = $column.DataType
+                            for ($j = 1; -not $name; $j++) {
+                                if ($table.Columns.ColumnName -notcontains "Column$j") { $name = "Column$j" }
                             }
-                            foreach ($row in $rows) {
-                                $dr = $dt.NewRow()
-                                foreach ($col in $row.Keys) {
-                                    $dr[$col] = $row[$col]
-                                }
-                                $null = $dt.Rows.Add($dr);
+                            $null = $table.Columns.Add($name, $datatype)
+                        }
+                        # read rows and assign values
+                        while ($reader.Read()) {
+                            $row = $table.NewRow()
+                            for ($i = 0; $i -lt $reader.FieldCount; $i++) {
+                                $row[$table.Columns[$i].ColumnName] = $reader.GetValue($i);
                             }
+                            $table.Rows.Add($row)
                         }
-                        $null = $ds.Tables.Add($dt);
+                        $ds.Tables.Add($table)
+                        $reader.Close()
+                        $reader.Dispose()
+                        $command.Dispose()
                     }
                 }
             }
@@ -269,7 +259,7 @@ function Invoke-DBOQuery {
             Stop-PSFFunction -EnableException $true -Message "Failed to run the query" -ErrorRecord $_
         }
         finally {
-            $managedConnection.Dispose()
+            $dataConnection.Dispose()
         }
         switch ($As) {
             'DataSet' {

functions/Register-DBOPackage.ps1

@@ -81,7 +81,7 @@ function Register-DBOPackage {
     .PARAMETER CreateDatabase
         Will create an empty database if missing on supported RDMBS
 
-    .PARAMETER ConnectionType
+    .PARAMETER Type
         Defines the driver to use when connecting to the database server.
         Available options: SqlServer (default), Oracle
 
@@ -153,9 +153,8 @@ function Register-DBOPackage {
         [switch]$CreateDatabase,
         [AllowNull()]
         [string]$ConnectionString,
-        [ValidateSet('SQLServer', 'Oracle')]
-        [Alias('Type', 'ServerType')]
-        [string]$ConnectionType = 'SQLServer'
+        [Alias('ConnectionType', 'ServerType')]
+        [DBOps.ConnectionType]$Type = (Get-DBODefaultSetting -Name rdbms.type -Value)
     )
 
     begin {
@@ -191,7 +190,7 @@ function Register-DBOPackage {
         }
         foreach ($key in ($PSBoundParameters.Keys)) {
             #If any custom properties were specified
-            if ($key -in @('OutputFile', 'Append', 'ConnectionType', 'Build')) {
+            if ($key -in @('OutputFile', 'Append', 'Type', 'Build')) {
                 $params += @{ $key = $PSBoundParameters[$key] }
             }
         }

functions/Test-DBOSupportedSystem.ps1

@@ -24,6 +24,18 @@ Function Test-DBOSupportedSystem {
     )
     begin {}
     process {
+        # try looking up already loaded assemblies
+        $lookupClass = switch ($Type) {
+            SqlServer { 'System.Data.SqlClient.SqlConnection' }
+            Oracle { 'Oracle.DataAccess.Client.OracleConnection' }
+            MySQL { 'MySql.Data.MySqlClient.MySqlConnection' }
+            PostgreSQL { 'Npgsql.NpgsqlConnection' }
+            default { Stop-PSFFunction -Message "Unknown type $Type" -EnableException $true }
+        }
+        if ([System.AppDomain]::CurrentDomain.GetAssemblies() | ForEach-Object { $_.GetType($lookupClass, 0) }) {
+            return $true
+        }
+        # otherwise get package from the local system
         $dependencies = Get-ExternalLibrary -Type $Type
         foreach ($package in $dependencies) {
             $packageEntry = Get-Package $package.Name -RequiredVersion $package.Version -ProviderName nuget -ErrorAction SilentlyContinue

internal/classes/DBOpsMySqlTableJournal.class.ps1

@@ -0,0 +1,13 @@
+class DBOpsMySqlTableJournal : DbUp.MySql.MySqlTableJournal {
+    # Constructors
+    DBOpsMySqlTableJournal([Func[DbUp.Engine.Transactions.IConnectionManager]] $connectionManager, [Func[DbUp.Engine.Output.IUpgradeLog]] $logger, [string] $schema, [string] $table) :base($connectionManager, $logger, $schema, $table) {}
+    # Overriding DoesTalbeExist method to get the proper schema table when schemaversiontable name is not specified
+    [string] DoesTableExistSql() {
+        if ($this.SchemaTableSchema) {
+            return "select 1 from INFORMATION_SCHEMA.TABLES where TABLE_NAME = '{0}' and TABLE_SCHEMA = '{1}'" -f $this.UnquotedSchemaTableName, $this.SchemaTableSchema
+        }
+        else {
+            return "select 1 from INFORMATION_SCHEMA.TABLES where TABLE_NAME = '{0}' and TABLE_SCHEMA = DATABASE()" -f $this.UnquotedSchemaTableName
+        }
+    }
+}

internal/functions/Get-ConnectionManager.ps1

@@ -1,7 +1,7 @@
 function Get-ConnectionManager {
     # Returns a connection manager object
     Param (
-        [Parameter(ParameterSetName = 'ConnString', Mandatory)]
+        [Parameter(ParameterSetName = 'ConnString')]
         [string]$ConnectionString,
         [Parameter(ParameterSetName = 'Config', Mandatory)]
         [DBOpsConfig]$Configuration,