From ede08686b9e67ff657976ece8d68f049e1c881b7 Mon Sep 17 00:00:00 2001 From: Yuval Neumann Date: Fri, 25 Jul 2025 19:38:01 +0300 Subject: [PATCH 1/4] Add initial Bidi support with configurable CCSID conversion - Added two new extension settings(index.ts): 1. Enable Bidi conversion 2. Target CCSID for Bidi conversion - Modified IBMiContent to apply Bidi-aware conversion logic when enabled via user settings, including fallback and error handling. --- src/api/IBMiContent.ts | 54 +++++++++++++++++++++++++++------- src/webviews/settings/index.ts | 2 ++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/api/IBMiContent.ts b/src/api/IBMiContent.ts index 7b1d779cd..c66861c08 100644 --- a/src/api/IBMiContent.ts +++ b/src/api/IBMiContent.ts @@ -170,25 +170,57 @@ export default class IBMiContent { const path = Tools.qualifyPath(library, sourceFile, member, asp, true); const tempRmt = this.getTempRemote(path); let retry = false; - while (true) { + + // BiDi handling if enabled + const isBidi = this.config.bidi; + const bidiCcsid = this.config.bidiCcsid; + + + while (true) { let copyResult: CommandResult; if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) { copyResult = { code: 0, stdout: '', stderr: '' }; - try { + + if (isBidi) { await this.ibmi.runSQL([ + `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, // @QSYS/CPYTOSTMF will fail if the remote temp already exists(with differenct ccsid) `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`, - `@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});` - ].join("\n")); - } catch (error: any) { - copyResult.code = -1; - copyResult.stderr = String(error); + `@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`, + `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);` + ].join("\n")).catch(e => { + copyResult.code = -1; + copyResult.stderr = String(e); + }); + } else { + try { + await this.ibmi.runSQL([ + `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`, + `@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});` + ].join("\n")); + } catch (error: any) { + copyResult.code = -1; + copyResult.stderr = String(error); + } } } else { - copyResult = await this.ibmi.runCommand({ - command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`, - noLibList: true - }); + if (isBidi) { + copyResult = { code: 0, stdout: '', stderr: '' }; + + await this.ibmi.runSQL([ + `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, // @QSYS/CPYTOSTMF will fail if the remote temp already exists(with differenct ccsid) + `@QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`, + `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);` + ].join("\n")).catch(e => { + copyResult.code = -1; + copyResult.stderr = String(e); + }); + } else { + copyResult = await this.ibmi.runCommand({ + command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`, + noLibList: true + }); + } } if (copyResult.code === 0) { diff --git a/src/webviews/settings/index.ts b/src/webviews/settings/index.ts index f5974bfca..fa9699629 100644 --- a/src/webviews/settings/index.ts +++ b/src/webviews/settings/index.ts @@ -114,6 +114,8 @@ export class SettingsUI { .addHorizontalRule() .addCheckbox(`enableSourceDates`, `Enable Source Dates`, `When enabled, source dates will be retained and updated when editing source members. Requires restart when changed.`, config.enableSourceDates) .addCheckbox(`sourceDateGutter`, `Source Dates in Gutter`, `When enabled, source dates will be displayed in the gutter. This also enables date search and sequence view.`, config.sourceDateGutter) + .addCheckbox(`bidi`,`Enable Bidi support for member editing`, `When enabled, source members encoded in Bidi CCSIDs will be automatically converted and displayed correctly during editing.`, config.bidi) + .addInput(`bidiCcsid`, `Bidi-compatible CCSID`, `The CCSID to use when converting bidirectional text members. Only applies when bidi support is enabled.`, { default: config.bidiCcsid || '', minlength: 0, maxlength: 5 }) .addHorizontalRule() .addSelect(`defaultDeploymentMethod`, `Default Deployment Method`, [ { From 3b2f2b73337da690bdf5211e55e8152749e65163 Mon Sep 17 00:00:00 2001 From: Yuval Neumann Date: Sat, 26 Jul 2025 12:47:15 +0300 Subject: [PATCH 2/4] Add Bidi support to include member upload Extends the Bidi handling to also apply when uploading source members back to the IBM i. --- src/api/IBMiContent.ts | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/api/IBMiContent.ts b/src/api/IBMiContent.ts index c66861c08..656c66824 100644 --- a/src/api/IBMiContent.ts +++ b/src/api/IBMiContent.ts @@ -183,7 +183,7 @@ export default class IBMiContent { if (isBidi) { await this.ibmi.runSQL([ - `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, // @QSYS/CPYTOSTMF will fail if the remote temp already exists(with differenct ccsid) + `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`, `@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`, `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);` @@ -208,7 +208,7 @@ export default class IBMiContent { copyResult = { code: 0, stdout: '', stderr: '' }; await this.ibmi.runSQL([ - `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, // @QSYS/CPYTOSTMF will fail if the remote temp already exists(with differenct ccsid) + `@QSYS/RMVLNK OBJLNK('${tempRmt}');`, `@QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`, `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);` ].join("\n")).catch(e => { @@ -285,6 +285,10 @@ export default class IBMiContent { const client = this.ibmi.client!; const tmpobj = await tmpFile(); + // BiDi handling if enabled + const isBidi = this.config.bidi; + const bidiCcsid = this.config.bidiCcsid; + try { await writeFileAsync(tmpobj, content || memberOrContent, `utf8`); const path = Tools.qualifyPath(library, sourceFile, member, asp, true); @@ -294,6 +298,19 @@ export default class IBMiContent { let copyResult: CommandResult; if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) { copyResult = { code: 0, stdout: '', stderr: '' }; + + if (isBidi) { + await this.ibmi.runSQL([ + `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`, + `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${bidiCcsid}) DTAFMT(*TEXT) REPLACE(*YES);`, + `@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});`, + `@QSYS/CPYF FROMFILE(QTEMP/QTEMPSRC) FROMMBR(TEMPMEMBER) TOFILE(${library}/${sourceFile}) TOMBR(${member}) MBROPT(*REPLACE);`, + `@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208);`, + ].join("\n")).catch(e => { + copyResult.code = -1; + copyResult.stderr = String(e); + }); + } else { try { await this.ibmi.runSQL([ `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) FROMMBR(${member}) TOFILE(QTEMP/QTEMPSRC) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`, @@ -305,12 +322,25 @@ export default class IBMiContent { copyResult.stderr = String(error); } } + } else { + if (isBidi) { + copyResult = { code: 0, stdout: '', stderr: '' }; + await this.ibmi.runSQL([ + `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${bidiCcsid}) DTAFMT(*TEXT) REPLACE(*YES);`, + `@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`, + `@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208);`, + ].join("\n")).catch(e => { + copyResult.code = -1; + copyResult.stderr = String(e); + }); + } else { copyResult = await this.ibmi.runCommand({ command: `QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`, noLibList: true }); } + } if (copyResult.code === 0) { const messages = Tools.parseMessages(copyResult.stderr); @@ -320,6 +350,7 @@ export default class IBMiContent { } return true; } else { + console.log(copyResult.command); throw new Error(`Failed uploading member: ${copyResult.stderr}`); } } catch (error) { From 68fd689b59294395f2d923ee45f033e480403484 Mon Sep 17 00:00:00 2001 From: Yuval Neumann Date: Sat, 26 Jul 2025 14:30:53 +0300 Subject: [PATCH 3/4] Update Bidi setting descriptions to clarify they are disabled when Source Dates are enabled --- src/webviews/settings/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webviews/settings/index.ts b/src/webviews/settings/index.ts index fa9699629..e8464dacb 100644 --- a/src/webviews/settings/index.ts +++ b/src/webviews/settings/index.ts @@ -114,7 +114,7 @@ export class SettingsUI { .addHorizontalRule() .addCheckbox(`enableSourceDates`, `Enable Source Dates`, `When enabled, source dates will be retained and updated when editing source members. Requires restart when changed.`, config.enableSourceDates) .addCheckbox(`sourceDateGutter`, `Source Dates in Gutter`, `When enabled, source dates will be displayed in the gutter. This also enables date search and sequence view.`, config.sourceDateGutter) - .addCheckbox(`bidi`,`Enable Bidi support for member editing`, `When enabled, source members encoded in Bidi CCSIDs will be automatically converted and displayed correctly during editing.`, config.bidi) + .addCheckbox(`bidi`,`Enable Bidi support for member editing`, `When enabled, members encoded in Bidi CCSIDs will be automatically converted and displayed correctly during editing. (Disabled when Source Dates are enabled.)`,config.bidi ) .addInput(`bidiCcsid`, `Bidi-compatible CCSID`, `The CCSID to use when converting bidirectional text members. Only applies when bidi support is enabled.`, { default: config.bidiCcsid || '', minlength: 0, maxlength: 5 }) .addHorizontalRule() .addSelect(`defaultDeploymentMethod`, `Default Deployment Method`, [ From d7b7870fbdc67db81a2aac1eedc9cd9bf43ec757 Mon Sep 17 00:00:00 2001 From: Itsanexpriment <44292066+Itsanexpriment@users.noreply.github.com> Date: Sat, 16 Aug 2025 18:39:36 +0300 Subject: [PATCH 4/4] test(bidi): add synchronous test suite in encoding.test.ts - Added BiDi test cases in src/api/tests/suites/encoding.test.ts - Covers Hebrew (CCSID 424) and Arabic (CCSID 420) - Includes helper convertToUTF8WithCCSID and edge cases --- src/api/tests/suites/encoding.test.ts | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/api/tests/suites/encoding.test.ts b/src/api/tests/suites/encoding.test.ts index 61dada617..8070dc7af 100644 --- a/src/api/tests/suites/encoding.test.ts +++ b/src/api/tests/suites/encoding.test.ts @@ -49,6 +49,31 @@ async function runCommandsWithCCSID(connection: IBMi, commands: string[], ccsid: } } +async function convertToUTF8WithCCSID(connection: IBMi, text: string, baseCcsid: string, intermediateCcsid: string): Promise { + const content = connection.getContent(); + const config = connection.getConfig(); + + const tempLib = config.tempLibrary; + const tempSPF = Tools.makeid(8); + const tempMbr = Tools.makeid(4); + + config.bidi = true; + config.bidiCcsid = intermediateCcsid; + + const createResult = await connection!.runCommand({ + command: `CRTSRCPF ${tempLib}/${tempSPF} MBR(${tempMbr}) CCSID(${baseCcsid})`, + environment: `ile`, + }); + console.log(`${tempLib}/${tempSPF} MBR(${tempMbr})`); + + const uploadResult = await content.uploadMemberContent(tempLib, tempSPF, tempMbr, text); + expect(uploadResult).toBeTruthy(); + + const memberContent = await content.downloadMemberContent(tempLib, tempSPF, tempMbr); + + return memberContent; +} + describe('Encoding tests', { concurrent: true }, () => { let connection: IBMi beforeAll(async () => { @@ -381,3 +406,54 @@ describe('Encoding tests', { concurrent: true }, () => { } }); }); + +// seperate suite for tests that require synchronous execution +// as global variables are modified in each test +describe('BiDi encoding tests', () => { + let connection: IBMi + beforeAll(async () => { + connection = await newConnection(); + }, CONNECTION_TIMEOUT); + + // test data + const BidiContents = { + "420": { + compatCcsid: "8612", + incompatCcsid: "273", + text: [ + "مرحبا بالعالم", + "Welcome to البرمجة", + "if a = 'با';", + "code then comment // مرحبا بالعالم", + ], + }, + "424": { + compatCcsid: "62211", + incompatCcsid: "273", + text: [ + "שלום עולם", + "English and Hebrew - עברית", + "if a = 'אר';", + "code then comment // הערה כלשהי", + ], + }, + }; + + const bidiEntries = Object.entries(BidiContents); + + bidiEntries.forEach(([baseCcsid, bidiContent]) => { + it(`Valid conversion of CCSID ${baseCcsid} to UTF8`, async () => { + const baseContent = bidiContent.text.join("\r\n") + "\r\n"; + const converted = await convertToUTF8WithCCSID(connection, baseContent, baseCcsid, bidiContent.compatCcsid); + expect(converted).toBe(baseContent); + }); + }); + + bidiEntries.forEach(([baseCcsid, bidiContent]) => { + it(`invalid conversion of CCSID ${baseCcsid} to UTF8`, async () => { + const baseContent = bidiContent.text.join("\r\n") + "\r\n"; + const converted = await convertToUTF8WithCCSID(connection, baseContent, baseCcsid, bidiContent.incompatCcsid); + expect(converted).not.toBe(baseContent); + }); + }); +}); \ No newline at end of file