Skip to content

Bidi CCSID Support via Extension Settings #2826

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Bidi CCSID Support via Extension Settings #2826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ede0868
Add initial Bidi support with configurable CCSID conversion
yuvalnn Jul 25, 2025
3b2f2b7
Add Bidi support to include member upload
yuvalnn Jul 26, 2025
68fd689
Update Bidi setting descriptions to clarify they are disabled when So…
yuvalnn Jul 26, 2025
2cf82bb
Merge branch 'master' into bidi-support
yuvalnn Aug 2, 2025
d7b7870
test(bidi): add synchronous test suite in encoding.test.ts
Itsanexpriment Aug 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 74 additions & 11 deletions src/api/IBMiContent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,25 +170,57 @@ export default class IBMiContent {
const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
const tempRmt = this.getTempRemote(path);
let retry = false;
while (true) {

// BiDi handling if enabled
const isBidi = this.config.bidi;
const bidiCcsid = this.config.bidiCcsid;


while (true) {
let copyResult: CommandResult;
if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) {
copyResult = { code: 0, stdout: '', stderr: '' };
try {

if (isBidi) {
await this.ibmi.runSQL([
`@QSYS/RMVLNK OBJLNK('${tempRmt}');`,
`@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`,
`@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});`
].join("\n"));
} catch (error: any) {
copyResult.code = -1;
copyResult.stderr = String(error);
`@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`,
`@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);`
].join("\n")).catch(e => {
copyResult.code = -1;
copyResult.stderr = String(e);
});
} else {
try {
await this.ibmi.runSQL([
`@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`,
`@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});`
].join("\n"));
} catch (error: any) {
copyResult.code = -1;
copyResult.stderr = String(error);
}
}
}
else {
copyResult = await this.ibmi.runCommand({
command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
noLibList: true
});
if (isBidi) {
copyResult = { code: 0, stdout: '', stderr: '' };

await this.ibmi.runSQL([
`@QSYS/RMVLNK OBJLNK('${tempRmt}');`,
`@QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`,
`@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES);`
].join("\n")).catch(e => {
copyResult.code = -1;
copyResult.stderr = String(e);
});
} else {
copyResult = await this.ibmi.runCommand({
command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
noLibList: true
});
}
}

if (copyResult.code === 0) {
Expand Down Expand Up @@ -253,6 +285,10 @@ export default class IBMiContent {
const client = this.ibmi.client!;
const tmpobj = await tmpFile();

// BiDi handling if enabled
const isBidi = this.config.bidi;
const bidiCcsid = this.config.bidiCcsid;

try {
await writeFileAsync(tmpobj, content || memberOrContent, `utf8`);
const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
Expand All @@ -262,6 +298,19 @@ export default class IBMiContent {
let copyResult: CommandResult;
if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) {
copyResult = { code: 0, stdout: '', stderr: '' };

if (isBidi) {
await this.ibmi.runSQL([
`@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`,
`@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${bidiCcsid}) DTAFMT(*TEXT) REPLACE(*YES);`,
`@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID});`,
`@QSYS/CPYF FROMFILE(QTEMP/QTEMPSRC) FROMMBR(TEMPMEMBER) TOFILE(${library}/${sourceFile}) TOMBR(${member}) MBROPT(*REPLACE);`,
`@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208);`,
].join("\n")).catch(e => {
copyResult.code = -1;
copyResult.stderr = String(e);
});
} else {
try {
await this.ibmi.runSQL([
`@QSYS/CPYF FROMFILE(${library}/${sourceFile}) FROMMBR(${member}) TOFILE(QTEMP/QTEMPSRC) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES);`,
Expand All @@ -273,12 +322,25 @@ export default class IBMiContent {
copyResult.stderr = String(error);
}
}
}
else {
if (isBidi) {
copyResult = { code: 0, stdout: '', stderr: '' };
await this.ibmi.runSQL([
`@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${bidiCcsid}) DTAFMT(*TEXT) REPLACE(*YES);`,
`@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(${bidiCcsid}) DBFCCSID(${this.config.sourceFileCCSID});`,
`@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208);`,
].join("\n")).catch(e => {
copyResult.code = -1;
copyResult.stderr = String(e);
});
} else {
copyResult = await this.ibmi.runCommand({
command: `QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
noLibList: true
});
}
}

if (copyResult.code === 0) {
const messages = Tools.parseMessages(copyResult.stderr);
Expand All @@ -288,6 +350,7 @@ export default class IBMiContent {
}
return true;
} else {
console.log(copyResult.command);
throw new Error(`Failed uploading member: ${copyResult.stderr}`);
}
} catch (error) {
Expand Down
76 changes: 76 additions & 0 deletions src/api/tests/suites/encoding.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,31 @@ async function runCommandsWithCCSID(connection: IBMi, commands: string[], ccsid:
}
}

async function convertToUTF8WithCCSID(connection: IBMi, text: string, baseCcsid: string, intermediateCcsid: string): Promise<string> {
const content = connection.getContent();
const config = connection.getConfig();

const tempLib = config.tempLibrary;
const tempSPF = Tools.makeid(8);
const tempMbr = Tools.makeid(4);

config.bidi = true;
config.bidiCcsid = intermediateCcsid;

const createResult = await connection!.runCommand({
command: `CRTSRCPF ${tempLib}/${tempSPF} MBR(${tempMbr}) CCSID(${baseCcsid})`,
environment: `ile`,
});
console.log(`${tempLib}/${tempSPF} MBR(${tempMbr})`);

const uploadResult = await content.uploadMemberContent(tempLib, tempSPF, tempMbr, text);
expect(uploadResult).toBeTruthy();

const memberContent = await content.downloadMemberContent(tempLib, tempSPF, tempMbr);

return memberContent;
}

describe('Encoding tests', { concurrent: true }, () => {
let connection: IBMi
beforeAll(async () => {
Expand Down Expand Up @@ -381,3 +406,54 @@ describe('Encoding tests', { concurrent: true }, () => {
}
});
});

// seperate suite for tests that require synchronous execution
// as global variables are modified in each test
describe('BiDi encoding tests', () => {
let connection: IBMi
beforeAll(async () => {
connection = await newConnection();
}, CONNECTION_TIMEOUT);

// test data
const BidiContents = {
"420": {
compatCcsid: "8612",
incompatCcsid: "273",
text: [
"مرحبا بالعالم",
"Welcome to البرمجة",
"if a = 'با';",
"code then comment // مرحبا بالعالم",
],
},
"424": {
compatCcsid: "62211",
incompatCcsid: "273",
text: [
"שלום עולם",
"English and Hebrew - עברית",
"if a = 'אר';",
"code then comment // הערה כלשהי",
],
},
};

const bidiEntries = Object.entries(BidiContents);

bidiEntries.forEach(([baseCcsid, bidiContent]) => {
it(`Valid conversion of CCSID ${baseCcsid} to UTF8`, async () => {
const baseContent = bidiContent.text.join("\r\n") + "\r\n";
const converted = await convertToUTF8WithCCSID(connection, baseContent, baseCcsid, bidiContent.compatCcsid);
expect(converted).toBe(baseContent);
});
});

bidiEntries.forEach(([baseCcsid, bidiContent]) => {
it(`invalid conversion of CCSID ${baseCcsid} to UTF8`, async () => {
const baseContent = bidiContent.text.join("\r\n") + "\r\n";
const converted = await convertToUTF8WithCCSID(connection, baseContent, baseCcsid, bidiContent.incompatCcsid);
expect(converted).not.toBe(baseContent);
});
});
});
2 changes: 2 additions & 0 deletions src/webviews/settings/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@ export class SettingsUI {
.addHorizontalRule()
.addCheckbox(`enableSourceDates`, `Enable Source Dates`, `When enabled, source dates will be retained and updated when editing source members. Requires restart when changed.`, config.enableSourceDates)
.addCheckbox(`sourceDateGutter`, `Source Dates in Gutter`, `When enabled, source dates will be displayed in the gutter. This also enables date search and sequence view.`, config.sourceDateGutter)
.addCheckbox(`bidi`,`Enable Bidi support for member editing`, `When enabled, members encoded in Bidi CCSIDs will be automatically converted and displayed correctly during editing. (Disabled when Source Dates are enabled.)`,config.bidi )
.addInput(`bidiCcsid`, `Bidi-compatible CCSID`, `The CCSID to use when converting bidirectional text members. Only applies when bidi support is enabled.`, { default: config.bidiCcsid || '', minlength: 0, maxlength: 5 })
.addHorizontalRule()
.addSelect(`defaultDeploymentMethod`, `Default Deployment Method`, [
{
Expand Down