Skip to content

Commit 0821ae6

Browse files
committed
Release 24.01
Code Renovation : Pipeline State Code Renovation: Copy State Improved Robustness for Pipeline state tracking and reporting Improved Robustness for Copy state tracking and reporting Improved Error Handling Fix #244
1 parent 3b1a728 commit 0821ae6

File tree

107 files changed

+2249
-1668
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

107 files changed

+2249
-1668
lines changed

docker/bin/runEverything.bat

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ goto :eof
88
if defined %1 set MSSQL14=%1
99
if not defined MSSQL14 set /p MSSQL14="SQL Server 2014 IP Address :"
1010
docker rm %CONTAINER_NAME%
11-
docker run --security-opt=seccomp:unconfined --name %CONTAINER_NAME% --memory="16g" -v YADAMU_01-SHARED:/usr/src/YADAMU/mnt --network YADAMU-NET -d -e YADAMU_TEST_NAME=everything -e TESTNAME=mssql2014TestSuite --add-host="MSSQL14-01:%MSSQL14%" yadamu/secure:latest
11+
docker run --security-opt=seccomp:unconfined --name %CONTAINER_NAME% --memory="16g" -v YADAMU_01-SHARED:/usr/src/YADAMU/mnt --network YADAMU-NET -e YADAMU_TEST_NAME=everything --add-host="MSSQL14-01:%MSSQL14%" -d yadamu/secure:latest
1212
docker logs %CONTAINER_NAME%
1313
exit /b
1414
:end
1515

1616
:windowsContainerSwarm
1717
@set MAX_MEMORY="4g"
1818
@set DISC_SIZE="100GB"
19-
docker run --name %CONTAINER_NAME% --network YADAMU-NET --storage-opt "size=%DISC_SIZE%" --memory %MAX_MEMORY% -v C:\SHARED:C:\YADAMU\mnt -e YADAMU_TEST_NAME=everything -e TESTNAME=mssql2014TestSuite -d yadamu/secure:latest
19+
docker run --name %CONTAINER_NAME% --network YADAMU-NET --storage-opt "size=%DISC_SIZE%" --memory %MAX_MEMORY% -v C:\SHARED:C:\YADAMU\mnt -e YADAMU_TEST_NAME=everything -d yadamu/secure:latest
2020
docker logs %CONTAINER_NAME%
2121
exit /b
2222
:end
@@ -25,7 +25,7 @@ goto :eof
2525
@set MAX_MEMORY="4g"
2626
@set DISC_SIZE="100GB"
2727
@set HOSTS=--add-host="VRTCA23-01:192.168.1.236" --add-host="VRTCA10-01:192.168.1.236" --add-host="azure-01:192.168.1.236" --add-host="minio-01:192.168.1.236"
28-
docker run --name %CONTAINER_NAME% --network YADAMU-NET --storage-opt "size=%DISC_SIZE%" --memory %MAX_MEMORY% -v C:\SHARED:C:\YADAMU\mnt %HOSTS% -e YADAMU_TEST_NAME=everything -e TESTNAME=mssql2014TestSuite -d yadamu/secure:latest
28+
docker run --name %CONTAINER_NAME% --network YADAMU-NET --storage-opt "size=%DISC_SIZE%" --memory %MAX_MEMORY% -v C:\SHARED:C:\YADAMU\mnt %HOSTS% -e YADAMU_TEST_NAME=everything -d yadamu/secure:latest
2929
docker logs %CONTAINER_NAME%
3030
exit /b
3131
:end
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
version: "2.4"
2+
services:
3+
4+
YADAMU-01:
5+
container_name: YADAMU-01
6+
image: yadamu/secure:latest
7+
entrypoint: sleep 365d
8+
mem_limit: '16g'
9+
security_opt:
10+
- seccomp:unconfined
11+
networks:
12+
- YADAMU-NET
13+
volumes:
14+
- YADAMU_01-SHARED:/usr/src/YADAMU/mnt
15+
restart: always
16+
17+
networks:
18+
YADAMU-NET:
19+
name: YADAMU-NET
20+
21+
22+
volumes:
23+
YADAMU_01-SHARED:
24+
name: YADAMU_01-SHARED
25+

docker/rdbms/mariadb/setup/configure.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
export DEBIAN_FRONTEND=noninteractive
33
apt-get update
44
apt-get install -y wget lsb-release gnupg
5+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B7B3B788A8D3785C
56
wget https://repo.mysql.com//mysql-apt-config_0.8.24-1_all.deb
67
dpkg -i mysql-apt-config_0.8.24-1_all.deb
78
apt-get update
8-
apt-get install -y mysql-shell
9+
apt-get install -y --allow-unauthenticated mysql-shell
910
export STAGE=/var/lib/mysql/stage
1011
cd $STAGE
1112
mkdir -p log

docker/regression/bin/chooseTest.ps1

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,12 +133,10 @@ switch ($ENV:YADAMU_TEST_NAME) {
133133
runRegressionTest "shortRegression"
134134
runRegressionTest "verticaShortRegression"
135135
runRegressionTest "vertica10ShortRegression"
136-
runRegressionTest "mssql2014ShortRegression"
137136
runRegressionTest "snowflakeShortRegression"
138137
runRegressionTest "postgresDataTypes"
139138
runRegressionTest "verticaDataTypes"
140139
runRegressionTest "vertica10DataTypes"
141-
runRegressionTest "mssql2014DataTypes"
142140
runRegressionTest "snowflakeDataTypes"
143141
runRegressionTest "export"
144142
runRegressionTest "import"
@@ -164,10 +162,12 @@ switch ($ENV:YADAMU_TEST_NAME) {
164162
runRegressionTest "mariadbCopy"
165163
runRegressionTest "verticaCopy"
166164
runRegressionTest "vertica10Copy"
167-
runRegressionTest "mssql2014TestSuite"
168165
runRegressionTest "snowflakeTestSuite"
169166
runRegressionTest "snowflakeCopy"
170167
runRegressionTest "snowflakeCopy"
168+
runRegressionTest "mssql2014ShortRegression"
169+
runRegressionTest "mssql2014DataTypes"
170+
runRegressionTest "mssql2014TestSuite"
171171
break
172172
}
173173

@@ -339,6 +339,16 @@ switch ($ENV:YADAMU_TEST_NAME) {
339339
}
340340
341341
"copy" {
342+
runRegressionTest "oracleCopy"
343+
runRegressionTest "postgresCopy"
344+
runRegressionTest "mysqlCopy"
345+
runRegressionTest "mariadbCopy"
346+
runRegressionTest "verticaCopy"
347+
runRegressionTest "vertica10Copy"
348+
break
349+
}
350+
351+
"copy2" {
342352
runRegressionTest "oracleCopy"
343353
runRegressionTest "postgresCopy"
344354
runRegressionTest "mysqlCopy"

docker/regression/bin/chooseTest.sh

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,10 @@ case $YADAMU_TEST_NAME in
5151
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh shortRegression
5252
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh verticaShortRegression
5353
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh vertica10ShortRegression
54-
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014ShortRegression
55-
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeShortRegression
54+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeShortRegression
5655
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh postgresDataTypes
5756
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh verticaDataTypes
5857
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh vertica10DataTypes
59-
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014DataTypes
6058
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeDataTypes
6159
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh export
6260
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh import
@@ -82,10 +80,12 @@ case $YADAMU_TEST_NAME in
8280
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mariadbCopy
8381
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh verticaCopy
8482
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh vertica10Copy
85-
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014TestSuite
8683
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeTestSuite
8784
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeCopy
8885
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh snowflakeCopy
86+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014ShortRegression
87+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014DataTypes
88+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mssql2014TestSuite
8989
;;
9090

9191
loader)
@@ -243,6 +243,15 @@ case $YADAMU_TEST_NAME in
243243
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mariadbCopy
244244
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh verticaCopy
245245
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh vertica10Copy
246+
;;
247+
248+
copy2)
249+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh oracleCopy
250+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh postgresCopy
251+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mysqlCopy
252+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mariadbCopy
253+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh verticaCopy
254+
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh vertica10Copy
246255
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh oracleCopy
247256
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh postgresCopy
248257
source $YADAMU_SCRIPT_DIR/runRegressionTest.sh mysqlCopy

src/node/core/dbReader.js

Lines changed: 48 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { pipeline, finished } from 'stream/promises';
66
import YadamuLibrary from '../lib/yadamuLibrary.js'
77
import YadamuConstants from '../lib/yadamuConstants.js'
88
import YadamuWriter from '../dbi/base/yadamuWriter.js'
9+
import DBIConstants from '../dbi/base/dbiConstants.js'
910

1011
import Yadamu from './yadamu.js'
1112
import {YadamuError, ExportError, DatabaseError, IterativeInsertError, InputStreamError} from './yadamuException.js'
@@ -116,13 +117,6 @@ class DBReader extends Readable {
116117
})
117118
}
118119

119-
setReader(pipeline) {
120-
const writer = pipeline.find((s) => { return s instanceof YadamuWriter})
121-
if (writer) {
122-
writer.setReader(pipeline[0])
123-
}
124-
}
125-
126120
async pipelineTable(task,readerDBI,writerDBI,retryOnError) {
127121

128122
retryOnError = retryOnError && (readerDBI.ON_ERROR === 'RETRY')
@@ -131,17 +125,15 @@ class DBReader extends Readable {
131125

132126
const yadamuPipeline = []
133127
const activeStreams = []
134-
let pipelineMetrics
128+
const pipelineState = DBIConstants.PIPELINE_STATE
135129

136130
try {
137131
queryInfo = readerDBI.generateSQLQuery(task)
138132
queryInfo.TARGET_DATA_TYPES = writerDBI.metadata?.[queryInfo.TABLE_NAME]?.dataTypes ?? []
139133
queryInfo.TARGET_COLUMN_NAMES = writerDBI.metadata?.[queryInfo.TABLE_NAME]?.columnNames ?? []
140-
const inputStreams = await readerDBI.getInputStreams(queryInfo,writerDBI.PARSE_DELAY)
141-
pipelineMetrics = inputStreams[0].COPY_METRICS
142-
yadamuPipeline.push(...inputStreams)
143-
const outputStreams = await writerDBI.getOutputStreams(queryInfo.MAPPED_TABLE_NAME,pipelineMetrics)
144-
yadamuPipeline.push(...outputStreams)
134+
const inputStreams = await readerDBI.getInputStreams(queryInfo,pipelineState)
135+
const outputStreams = await writerDBI.getOutputStreams(queryInfo.MAPPED_TABLE_NAME,pipelineState)
136+
yadamuPipeline.push(...inputStreams,...outputStreams)
145137
activeStreams.push(...yadamuPipeline.map((s) => { return finished(s) }))
146138
} catch (e) {
147139
this.yadamuLogger.handleException(['PIPELINE','STREAM INITIALIZATION',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,task.TABLE_NAME],e)
@@ -156,18 +148,21 @@ class DBReader extends Readable {
156148
})
157149
this.activeReaders.add(activeReader)
158150
// Pass the Reader to the YadamuWriter instance so it can calculate lost rows correctly in the event of an error
159-
this.setReader(yadamuPipeline)
151+
160152
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],`${yadamuPipeline.map((s) => { return s.constructor.name }).join(' => ')}`)
161-
162153
// this.traceStreamEvents(yadamuPipeline,queryInfo.TABLE_NAME)
163154

164-
pipelineMetrics.pipeStartTime = performance.now();
155+
pipelineState.startTime = performance.now();
165156
await pipeline(...yadamuPipeline)
166-
pipelineMetrics.pipeEndTime = performance.now();
157+
pipelineState.endTime = performance.now();
158+
159+
// Report Pipeline state
160+
// console.log(pipelineState)
161+
writerDBI.reportPipelineStatus(pipelineState)
162+
167163
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],`${yadamuPipeline.map((s) => { return `${s.constructor.name}:${s.destroyed}` }).join(' => ')}`)
168-
169-
170164
} catch (err) {
165+
pipelineState.endTime = performance.now();
171166

172167
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME,readerDBI.ON_ERROR,'FAILED'],`${err.constructor.name},${err.message}`)
173168

@@ -181,18 +176,18 @@ class DBReader extends Readable {
181176
throw err
182177
}
183178

184-
// Wait for all components of the pipeline to finish before closing connections
185-
179+
// Wait for all components of the pipeline to finish before closing connections
186180
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE','FAILED','STREAMS_COMPLETE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,readerDBI.ON_ERROR,readerDBI.getWorkerNumber(),task.TABLE_NAME,`${yadamuPipeline.map((s) => { return `${s.constructor.name}`}).join(' => ')}`],'WAITING')
187181
await Promise.allSettled(activeStreams)
188182
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE','FAILED','STREAMS_COMPLETE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,readerDBI.ON_ERROR,readerDBI.getWorkerNumber(),task.TABLE_NAME,`${yadamuPipeline.map((s) => { return `${s.constructor.name}`}).join(' => ')}`],'COMPLETED')
189-
190-
// console.log(pipelineMetrics)
191-
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],`${yadamuPipeline.map((s) => { return `${s.constructor.name}:[${s.readableLength},${s.writableLength}]` }).join(',')}`)
192183

193-
// Determine the underlying cause of the error.
194-
const cause = pipelineMetrics.readerError || pipelineMetrics.parserError || yadamuPipeline.find((s) => {return s.underlyingError instanceof Error})?.underlyingError || err
195-
184+
185+
// Report Pipeline state and determine the underlying cause of the error.
186+
// console.log(pipelineState)
187+
let cause = writerDBI.reportPipelineStatus(pipelineState,err)
188+
189+
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],`${yadamuPipeline.map((s) => { return `${s.constructor.name}:[${s.readableLength},${s.writableLength}]` }).join(',')}`)
190+
196191
// Verify all components of the pipeline have been destroyed.
197192
// this.yadamuLogger.trace([this.constructor.name,'PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],`${yadamuPipeline.map((s) => { return `${s.constructor.name}:${s.destroyed}` }).join(' => ')}`)
198193
// yadamuPipeline.map((s) => { if (!s.destroyed){ s.destroy(cause)})
@@ -202,56 +197,61 @@ class DBReader extends Readable {
202197
throw cause;
203198
}
204199

205-
if (YadamuError.lostConnection(pipelineMetrics.readerError) || YadamuError.lostConnection(pipelineMetrics.parserError)) {
200+
if (YadamuError.lostConnection(pipelineState[DBIConstants.INPUT_STREAM_ID].error) || YadamuError.lostConnection(pipelineState[DBIConstants.PARSER_STREAM_ID].error)) {
206201
// If the reader or parser failed with a lost connection error re-establish the input stream connection
207202
await readerDBI.reconnect(cause,'READER')
208203

209204
}
210205
else {
211206
this.yadamuLogger.handleException(['PIPELINE',readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,queryInfo.MAPPED_TABLE_NAME],cause)
212207
}
213-
208+
214209
if (retryOnError) {
215-
await this.retryPipelineTable(task,readerDBI,writerDBI,pipelineMetrics)
210+
await this.retryPipelineTable(task,readerDBI,writerDBI,pipelineState)
216211
}
217212

218-
this.dbi.resetExceptionTracking()
219213
}
220214

221-
return pipelineMetrics
215+
return pipelineState
222216

223217
}
224218

225-
pipelineSucceeded(metrics) {
226-
return (metrics.read === metrics.committed)
219+
pipelineSucceeded(pipelineState) {
220+
return (pipelineState.read === pipelineState.committed)
227221
}
228222

229-
duplicateCause(originalMetrics,latestMetrics) {
230-
return (originalMetrics.readerError === latestMetrics.readerError)
231-
&& (originalMetrics.parserError === latestMetrics.parserError)
232-
&& (originalMetrics.managerError === latestMetrics.managerError)
233-
&& (originalMetrics.writerError === latestMetrics.writerError)
223+
isDuplicateException(a,b){
224+
225+
return ((a instanceof Error) &&
226+
(b instanceof Error) &&
227+
(a.message === b.message))
228+
229+
}
230+
231+
duplicateCause(previousState,currentState) {
232+
return this.isDuplicateException(previousState[DBIConstants.INPUT_STREAM_ID].error,currentState[DBIConstants.INPUT_STREAM_ID].error)
233+
|| this.isDuplicateException(previousState[DBIConstants.PARSER_STREAM_ID].error,currentState[DBIConstants.PARSER_STREAM_ID].error)
234+
|| this.isDuplicateException(previousState[DBIConstants.TRANSFORMATION_STREAM_ID].error,currentState[DBIConstants.TRANSFORMATION_STREAM_ID].error)
235+
|| this.isDuplicateException(previousState[DBIConstants.OUTPUT_STREAM_ID].error,currentState[DBIConstants.OUTPUT_STREAM_ID].error)
234236
}
235237

236-
async retryPipelineTable(task,readerDBI,writerDBI,originalMetrics) {
238+
async retryPipelineTable(task,readerDBI,writerDBI,previousState) {
237239

238240
/*
239241
**
240242
** Retry the operation. If the operation up to RETRY_COUNT times. If the operation fails with a similar exception ABORT.
241243
**
242244
*/
243-
244245
let retryCount = 0
245-
let retryMetrics = originalMetrics
246+
let currentState = previousState
246247
do {
247-
const errorType = readerDBI.raisedError(retryMetrics.readerError) ? 'READER ERROR' : writerDBI.raisedError(retryMetrics.readerError) ? 'WRITER ERROR' : 'TRANSFORM ERROR'
248+
const errorType = readerDBI.raisedError(currentState[DBIConstants.INPUT_STREAM_ID].error) ? 'READER ERROR' : writerDBI.raisedError(currentState[DBIConstants.OUTPUT_STREAM_ID].error) ? 'WRITER ERROR' : 'TRANSFORM ERROR'
248249
this.yadamuLogger.info(['PIPELINE',errorType,readerDBI.DATABASE_VENDOR,writerDBI.DATABASE_VENDOR,task.TABLE_NAME,readerDBI.ON_ERROR],`Retrying Operation.`)
249250
await writerDBI.truncateTable(writerDBI.CURRENT_SCHEMA,task.TABLE_NAME)
250251
readerDBI.adjustQuery(task)
251252
retryCount++
252-
retryMetrics = await this.pipelineTable(task,readerDBI,writerDBI,false)
253-
} while (!this.pipelineSucceeded(retryMetrics) && !this.duplicateCause(originalMetrics,retryMetrics) && (retryCount < 6))
254-
253+
currentState = await this.pipelineTable(task,readerDBI,writerDBI,false)
254+
} while (!this.pipelineSucceeded(currentState) && !this.duplicateCause(previousState,currentState) && (retryCount < 6))
255255
}
256256

257257
async pipelineTables(taskList,readerDBI,writerDBI) {
@@ -303,7 +303,7 @@ class DBReader extends Readable {
303303
return [this]
304304
}
305305
else {
306-
return this.dbi.getInputStreams()
306+
return this.dbi.getInputStreams(DBIConstants.PIPELINE_STATE)
307307
}
308308

309309
}
@@ -373,10 +373,7 @@ class DBReader extends Readable {
373373
async doDestroy(err) {
374374
// this.yadamuLogger.trace([this.constructor.name,`destroy`,this.dbi.DATABASE_VENDOR],``)
375375
try {
376-
// Try to clean up the DBI gracefully
377-
// this.yadamuLogger.trace([this.constructor.name,this.nextPhase],'DDL COMPLETE: WAITING')
378-
// await this.dbWriter.dbi.ddlComplete
379-
// this.yadamuLogger.trace([this.constructor.name,this.nextPhase],'DDL COMPLETE: PROCESSING')
376+
await this.dbi.finalizeRead()
380377
await this.dbi.finalizeExport();
381378
await this.dbi.final()
382379
}

0 commit comments

Comments
 (0)