Release 2024-02

markddrake · markddrake · commit c1603d8ae6d6 · 2024-02-14T09:18:35.000-08:00
Added SQL Server 2022 Command Line Tests
Fixed invalid JSON in mariadbDataTypes.json
Add pipeline state information to exceptions that occur during pipeline operations
Fix exception handling in OracleQA driver
diff --git a/qa/cmdLine/mssql22/bin/cmdLineTests.bat b/qa/cmdLine/mssql22/bin/cmdLineTests.bat
@@ -0,0 +1 @@
+@call %~dp0..\..\bin\cmdLineTests.bat %0
diff --git a/qa/cmdLine/mssql22/bin/cmdLineTests.sh b/qa/cmdLine/mssql22/bin/cmdLineTests.sh
@@ -0,0 +1 @@
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/bin/cmdLineTests.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/export_sample_datasets.bat b/qa/cmdLine/mssql22/bin/export_sample_datasets.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\export_sample_datasets.bat %0
diff --git a/qa/cmdLine/mssql22/bin/export_sample_datasets.sh b/qa/cmdLine/mssql22/bin/export_sample_datasets.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/export_sample_datasets.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/import_MsSQL.bat b/qa/cmdLine/mssql22/bin/import_MsSQL.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\import_MsSQL.bat %0
diff --git a/qa/cmdLine/mssql22/bin/import_MsSQL.sh b/qa/cmdLine/mssql22/bin/import_MsSQL.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/import_MsSQL.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/import_MySQL.bat b/qa/cmdLine/mssql22/bin/import_MySQL.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from test root directory
+REM
+@call %~dp0..\..\mssql\bin\import_MySQL.bat %0
diff --git a/qa/cmdLine/mssql22/bin/import_MySQL.sh b/qa/cmdLine/mssql22/bin/import_MySQL.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/import_MySQL.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/import_Oracle.bat b/qa/cmdLine/mssql22/bin/import_Oracle.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\import_Oracle.bat %0
diff --git a/qa/cmdLine/mssql22/bin/import_Oracle.sh b/qa/cmdLine/mssql22/bin/import_Oracle.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/import_Oracle.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/upload_MsSQL.bat b/qa/cmdLine/mssql22/bin/upload_MsSQL.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\upload_MsSQL.bat %0
diff --git a/qa/cmdLine/mssql22/bin/upload_MsSQL.sh b/qa/cmdLine/mssql22/bin/upload_MsSQL.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/upload_MsSQL.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/upload_MySQL.bat b/qa/cmdLine/mssql22/bin/upload_MySQL.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\upload_MySQL.bat %0
diff --git a/qa/cmdLine/mssql22/bin/upload_MySQL.sh b/qa/cmdLine/mssql22/bin/upload_MySQL.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/upload_MySQL.sh ${BASH_SOURCE[0]}
diff --git a/qa/cmdLine/mssql22/bin/upload_Oracle.bat b/qa/cmdLine/mssql22/bin/upload_Oracle.bat
@@ -0,0 +1,4 @@
+REM
+REM Must be run from YADAMU home folder
+REM
+@call %~dp0..\..\mssql\bin\upload_Oracle.bat %0
diff --git a/qa/cmdLine/mssql22/bin/upload_Oracle.sh b/qa/cmdLine/mssql22/bin/upload_Oracle.sh
@@ -0,0 +1,4 @@
+#
+# Must be run from test root directory
+#
+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/mssql/bin/upload_Oracle.sh ${BASH_SOURCE[0]}
diff --git a/qa/regression/initialize.json b/qa/regression/initialize.json
@@ -5,7 +5,6 @@
 , "recreateSchema"     : true
 , "targetSchemaSuffix" : ""
 , "parameters": {
-	   "SQL_TRACE" : "trace.sql"
   }
 , "tests"                  : [{
     "source": "oracle18c"
diff --git a/qa/regression/mariadbDataTypes.json b/qa/regression/mariadbDataTypes.json
@@ -12,7 +12,7 @@
     "operation"       : "import"
   , "source"          : "file-staging"
   , "targets"         : [
-       "mariadb#1
+       "mariadb#1"
 	]
   , "verifyOperation" : true
   , "parameters"      : {
diff --git a/src/node/core/yadamuException.js b/src/node/core/yadamuException.js
@@ -17,6 +17,28 @@ class YadamuError extends Error {
 	return ((e instanceof DatabaseError) && e.missingTable())
   }
      
+  // Create a simple clone of the pipeline state that ensures there are no recursive structures when appending currrent pipeline state to an error
+  
+  static clonePipelineState(state) {
+	 return {
+	   failed            : state.failed       
+	 , startTime         : state.startTime    
+     , endTime           : state.endTime      
+ 	 , read              : state.read         
+	 , parsed            : state.parsed       
+	 , received          : state.received     
+     , committed         : state.committed    
+     , cached            : state.cached       
+	 , pending           : state.pending      
+     , written           : state.written      
+     , skipped           : state.skipped      
+     , lost              : state.lost         
+	 , idleTime          : state.idleTime     
+     , batchNumber       : state.batchNumber  
+     , batchWritten      : state.batchWritten 
+	}
+  }	  
+	 
   static createIterativeException(dbi,tableInfo,cause,batchSize,rowNumber,row,info) {
 
 	const tableName = tableInfo.tableName
diff --git a/src/node/dbi/base/yadamuDBI.js b/src/node/dbi/base/yadamuDBI.js
@@ -1122,7 +1122,7 @@ class YadamuDBI extends EventEmitter {
     */
  
     if (this.IS_WRITER === true) {
-      if (this.PIPELINE_STATE.written > 0) {
+      if (this.PIPELINE_STATE?.written > 0) {
         this.LOGGER.error([`RECONNECT`,this.DATABASE_VENDOR],`${this.PIPELINE_STATE.written} uncommitted rows discarded when connection lost.`)
         this.PIPELINE_STATE.lost += this.PIPELINE_STATE.written;
         this.PIPELINE_STATE.written = 0;
diff --git a/src/node/dbi/base/yadamuOutputManager.js b/src/node/dbi/base/yadamuOutputManager.js
@@ -1,16 +1,29 @@
 
-import fs              from 'fs'
-import assert          from 'assert'
-import { performance } from 'perf_hooks';
-
-import {Readable, Writable, Transform, pipeline } from 'stream'
-
-import YadamuLibrary   from '../../lib/yadamuLibrary.js';
-import YadamuConstants from '../../lib/yadamuConstants.js';
-
-import DBIConstants    from './dbiConstants.js'
-
-import {YadamuError, BatchInsertError, IterativeInsertError, DatabaseError, InvalidMessageSequence} from '../../core/yadamuException.js'
+import fs                from 'fs'
+import assert            from 'assert'
+import { 
+  performance 
+}                        from 'perf_hooks';
+
+import {
+  Readable, 
+  Writable, 
+  Transform, 
+  pipeline 
+}                        from 'stream'
+
+import YadamuLibrary     from '../../lib/yadamuLibrary.js';
+import YadamuConstants   from '../../lib/yadamuConstants.js';
+
+import DBIConstants      from './dbiConstants.js'
+
+import {
+  YadamuError, 
+  BatchInsertError, 
+  IterativeInsertError, 
+  DatabaseError, 
+  InvalidMessageSequence
+}                        from '../../core/yadamuException.js'
 
 class YadamuOutputManager extends Transform {
 
@@ -430,6 +443,7 @@ class YadamuOutputManager extends Transform {
 	  this.PIPELINE_STATE.failed = true
       this.PIPELINE_STATE.errorSource = this.PIPELINE_STATE.errorSource || DBIConstants.TRANSFORMATION_STREAM_ID
 	  this.STREAM_STATE.error = this.STREAM_STATE.error || (this.PIPELINE_STATE.errorSource === DBIConstants.TRANSFORMATION_STREAM_ID) ? err : this.STREAM_STATE.error
+	  err.pipelineState = YadamuError.clonePipelineState(this.PIPELINE_STATE)
     }
 	
 	try {
diff --git a/src/node/dbi/base/yadamuParser.js b/src/node/dbi/base/yadamuParser.js
@@ -1,18 +1,22 @@
 
 import { 
   performance
-}                       from 'perf_hooks';
+}                        from 'perf_hooks';
 
 import {
   setTimeout 
-}                       from 'timers/promises'
+}                        from 'timers/promises'
 
 import { 
   Transform 
-}                       from 'stream';
+}                        from 'stream';
 
 import DBIConstants      from './dbiConstants.js';
 
+import {
+  YadamuError
+}                        from '../../core/yadamuException.js'
+
 class YadamuParser extends Transform {
 
   get LOGGER()               { return this._LOGGER }
@@ -140,6 +144,7 @@ class YadamuParser extends Transform {
       this.PIPELINE_STATE.failed = true
       this.PIPELINE_STATE.errorSource = this.PIPELINE_STATE.errorSource || DBIConstants.PARSER_STREAM_ID
 	  this.STREAM_STATE.error = this.STREAM_STATE.error || (this.PIPELINE_STATE.errorSource === DBIConstants.PARSER_STREAM_ID) ? err : this.STREAM_STATE.error
+	  err.pipelineState = YadamuError.clonePipelineState(this.PIPELINE_STATE)
     }  
 
   }
diff --git a/src/node/dbi/base/yadamuWriter.js b/src/node/dbi/base/yadamuWriter.js
@@ -21,7 +21,8 @@ import {
   YadamuError, 
   BatchInsertError, 
   IterativeInsertError, 
-  DatabaseError}         from '../../core/yadamuException.js'
+  DatabaseError
+}                        from '../../core/yadamuException.js'
 
 
 class YadamuWriter extends Writable {
@@ -467,6 +468,7 @@ class YadamuWriter extends Writable {
       this.PIPELINE_STATE.failed = true
       this.PIPELINE_STATE.errorSource = this.PIPELINE_STATE.errorSource || DBIConstants.OUTPUT_STREAM_ID
       this.STREAM_STATE.error = this.STREAM_STATE.error || (this.PIPELINE_STATE.errorSource === DBIConstants.OUTPUT_STREAM_ID) ? err : this.STREAM_STATE.error
+	  err.pipelineState = YadamuError.clonePipelineState(this.PIPELINE_STATE)
       
       try {
         // this.LOGGER.trace([this.constructor.name,'doDestroy()','BATCH_COMPLETE',this.dbi.getWorkerNumber(),this.tableName],'WAITING')
diff --git a/src/package.json b/src/package.json
@@ -16,7 +16,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.441.0",
     "@aws-sdk/lib-storage": "^3.441.0",
-    "@azure/storage-blob": "12.16",
+    "@azure/storage-blob": "^12.17.0",
     "@electron/remote": "^2.0.12",
     "bootstrap": "^5.3.2",
     "bootstrap-icons": "^1.11.1",
diff --git a/src/qa/dbi/oracle/oracleQA.js b/src/qa/dbi/oracle/oracleQA.js
@@ -47,7 +47,7 @@ class OracleQA extends YadamuQALibrary.qaMixin(OracleDBI) {
           await this.executeSQL(dropUser,{});      
 		  break;
         } catch (e) {
-		  if (e.lockingError() && (retryCount > 0)) {
+		  if ((e instanceof OracleError) && e.lockingError() && (retryCount > 0)) {
 			await setTimeout(1000)
 			retryCount--
 			continue

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+@call %~dp0..\..\bin\cmdLineTests.bat %0`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+source $(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")/bin/cmdLineTests.sh ${BASH_SOURCE[0]}`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +REM
 +REM Must be run from YADAMU home folder
 +REM
 +@call %~dp0..\..\mssql\bin\export_sample_datasets.bat %0