Bidi CCSID Support via Extension Settings

src/api/IBMi.ts

@@ -16,7 +16,7 @@ import { ConnectionManager } from './configuration/config/ConnectionManager';
 import { ConnectionConfig, RemoteConfigFile } from './configuration/config/types';
 import { ConfigFile } from './configuration/serverFile';
 import { CachedServerSettings, CodeForIStorage } from './configuration/storage/CodeForIStorage';
-import { AspInfo, CommandData, CommandResult, ConnectionData, EditorPath, IBMiMember, RemoteCommand } from './types';
+import { AspInfo, CommandData, CommandResult, ConnectionData, EditorPath, IBMiMember, RemoteCommand, QsysPath, CacheItem } from './types';
 
 export interface MemberParts extends IBMiMember {
   basename: string
@@ -147,6 +147,8 @@ export default class IBMi {
   //Maximum admited length for command's argument - any command whose arguments are longer than this won't be executed by the shell
   maximumArgsLength = 0;
 
+  ccsidCache: Map<string, CacheItem<number>> = new Map();
+
   public appendOutput: (text: string) => void = (text) => {
     process.stdout.write(text);
   };
@@ -1520,6 +1522,51 @@ export default class IBMi {
     }
   }
 
+  async getFileCcsid(path: string | QsysPath): Promise<number> {
+    const minute = 60 * 1000;
+    const timeToLive = 10 * minute;
+
+    const isQsysPath = typeof path === `object`;
+    let fileObjPath: string;
+
+    if (isQsysPath) {
+      const localPath: QsysPath = path;
+      localPath.asp = localPath.asp ? this.sysNameInAmerican(localPath.asp) : undefined;
+      localPath.library = this.sysNameInAmerican(localPath.library);
+      localPath.name = this.sysNameInAmerican(localPath.name);
+      fileObjPath = Tools.qualifyPath(localPath.library, localPath.name, '', localPath.asp || '', true);
+    } else {
+      fileObjPath = path.replace(/\/[^/]+\.MBR$/i, ''); // strip the member from path
+    }
+
+    const cached = this.ccsidCache.get(fileObjPath);
+    if (cached) {
+      if (!cached.createdAt || cached.createdAt + timeToLive >= Date.now()) {
+        // cached ccsid still valid
+        return cached.value;
+      }
+      // clear the stale cached ccsid
+      this.ccsidCache.delete(fileObjPath);
+    }
+
+    // Take {DOES_THIS_WORK: `YESITDOES`} away, and all of a sudden names with # aren't found.
+    const result = await this.sendCommand({ command: `${this.remoteFeatures.attr} -p "${fileObjPath}" CCSID`, env: { DOES_THIS_WORK: `YESITDOES` } });
+
+    if (result.code === 0) {
+      const pieces = result.stdout.split('=');
+      if (pieces.length === 2) {
+        const ccsid = Number(pieces[1]) || 0;
+        if (ccsid !== 0) {
+          // save result to cache
+          this.ccsidCache.set(fileObjPath, { value: ccsid, createdAt: Date.now() })
+          return ccsid;
+        }
+      }
+    }
+
+    return 0;
+  }
+
   getLibraryIAsp(library: string) {
     const found = this.libraryAsps.get(library);
     if (found && found >= 0) {

src/api/IBMiContent.ts

@@ -152,10 +152,27 @@ export default class IBMiContent {
     const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
     const tempRmt = this.getTempRemote(path);
     let retry = false;
-    while (true) {
+
+    // Fetch source file CCSID and determine if conversion is needed
+    const sourceCcsid = await this.ibmi.getFileCcsid(path);
+    const {requiresConversion, targetCcsid} = Tools.determineCcsidConversion(sourceCcsid, this.config);
+
+  while (true) {
       let copyResult: CommandResult;
       if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) {
         copyResult = { code: 0, stdout: '', stderr: '' };
+        if (requiresConversion) {
+          await this.ibmi.runSQL(`@QSYS/RMVLNK OBJLNK('${tempRmt}')`).catch(_ => {}); // ignore error
+          await this.ibmi.runSQL([
+            `Drop table if exists QTEMP.QTEMPSRC`,
+            `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES)`,
+            `@QSYS/CPYTOSTMF FROMMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${targetCcsid}) DBFCCSID(${this.config.sourceFileCCSID})`,
+            `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES)`
+          ].join("\n")).catch(e => {
+            copyResult.code = -1;
+            copyResult.stderr = String(e);
+          });
+        } else {
         try {
           await this.ibmi.runSQL([
             `Drop table if exists QTEMP.QTEMPSRC`,
@@ -165,12 +182,28 @@ export default class IBMiContent {
           copyResult.code = -1;
           copyResult.stderr = String(error);
         }
+        }
       }
       else {
-        copyResult = await this.ibmi.runCommand({
-          command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
-          noLibList: true
-        });
+        if (requiresConversion) {
+          copyResult = { code: 0, stdout: '', stderr: '' };
+
+          await this.ibmi.runSQL(`@QSYS/RMVLNK OBJLNK('${tempRmt}')`).catch(_ => {}); // ignore error
+          try {
+            await this.ibmi.runSQL([
+              `@QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(${targetCcsid}) DBFCCSID(${this.config.sourceFileCCSID})`,
+              `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(1208) DTAFMT(*TEXT) REPLACE(*YES)`
+            ]);
+          } catch (e: any) {
+            copyResult.code = -1;
+            copyResult.stderr = String(e);
+          }
+        } else {
+          copyResult = await this.ibmi.runCommand({
+            command: `QSYS/CPYTOSTMF FROMMBR('${path}') TOSTMF('${tempRmt}') STMFOPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
+            noLibList: true
+          });
+        }
       }
 
       if (copyResult.code === 0) {
@@ -221,6 +254,10 @@ export default class IBMiContent {
     try {
       await writeFileAsync(tmpobj, content, `utf8`);
       const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
+      // Fetch source file CCSID and determine if conversion is needed
+      const sourceCcsid = await this.ibmi.getFileCcsid(path);
+      const {requiresConversion, targetCcsid} = Tools.determineCcsidConversion(sourceCcsid, this.config);
+
       const tempRmt = this.getTempRemote(path);
 
       const touchUnicode = await this.ibmi.sendCommand({ command: `touch ${tempRmt} && attr ${tempRmt} CCSID=1208` });
@@ -232,6 +269,21 @@ export default class IBMiContent {
       let copyResult: CommandResult;
       if (this.ibmi.dangerousVariants && new RegExp(`[${this.ibmi.variantChars.local}]`).test(path)) {
         copyResult = { code: 0, stdout: '', stderr: '' };
+
+        if (requiresConversion) {
+          try {
+          await this.ibmi.runSQL([
+            `@QSYS/CPYF FROMFILE(${library}/${sourceFile}) TOFILE(QTEMP/QTEMPSRC) FROMMBR(${member}) TOMBR(TEMPMEMBER) MBROPT(*REPLACE) CRTFILE(*YES)`,
+            `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${targetCcsid}) DTAFMT(*TEXT) REPLACE(*YES)`,
+            `@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${Tools.qualifyPath("QTEMP", "QTEMPSRC", "TEMPMEMBER", undefined)}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
+            `@QSYS/CPYF FROMFILE(QTEMP/QTEMPSRC) FROMMBR(TEMPMEMBER) TOFILE(${library}/${sourceFile}) TOMBR(${member}) MBROPT(*REPLACE)`,
+            `@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208)`
+            ].join("\n"));
+          } catch (e: any) {
+            copyResult.code = -1;
+            copyResult.stderr = String(e);
+          }
+        } else {
         try {
           await this.ibmi.runSQL([
             `Drop table if exists QTEMP.QTEMPSRC`,
@@ -244,12 +296,27 @@ export default class IBMiContent {
           copyResult.stderr = String(error);
         }
       }
+      }
       else {
+         if (requiresConversion) {
+          copyResult = { code: 0, stdout: '', stderr: '' };
+          try {
+            await this.ibmi.runSQL([
+            `@QSYS/CPY OBJ('${tempRmt}') TOOBJ('${tempRmt}') TOCCSID(${targetCcsid}) DTAFMT(*TEXT) REPLACE(*YES)`,
+            `@QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(${targetCcsid}) DBFCCSID(${this.config.sourceFileCCSID})`,
+            `@CHGATR OBJ('${tempRmt}') ATR(*CCSID) VALUE(1208)`
+            ]);
+          } catch (e: any) {
+            copyResult.code = -1;
+            copyResult.stderr = String(e);
+          }
+        } else {
         copyResult = await this.ibmi.runCommand({
           command: `QSYS/CPYFRMSTMF FROMSTMF('${tempRmt}') TOMBR('${path}') MBROPT(*REPLACE) STMFCCSID(1208) DBFCCSID(${this.config.sourceFileCCSID})`,
           noLibList: true
         });
       }
+      }
 
       if (copyResult.code === 0) {
         const messages = Tools.parseMessages(copyResult.stderr);

src/api/Tools.ts

@@ -187,4 +187,34 @@ export namespace Tools {
     path = path.replace("~", os.homedir());
     return process.platform === "win32" ? path.replaceAll('/', '\\') : path;
   }
+
+  /**
+   * Determines if CCSID conversion should be applied based on configuration and source file CCSID.
+   * @param sourceCcsid The CCSID of the source file
+   * @param config Connection configuration containing conversion settings
+   * @returns {requiresConversion, targetCcsid} - whether conversion is required and the target CCSID to use
+   */
+  export function determineCcsidConversion(sourceCcsid: number, config: { ccsidConversionEnabled?: boolean, ccsidConvertFrom?: string, ccsidConvertTo?: string }): { requiresConversion: boolean, targetCcsid: number } {
+    const noConversion = { requiresConversion: false, targetCcsid: 0 };
+
+    // conversion must be enabled
+    if (!config.ccsidConversionEnabled) {
+      return noConversion;
+    }
+
+    const configuredSourceCcsid = Number(config.ccsidConvertFrom) || 0;
+    const configuredTargetCcsid = Number(config.ccsidConvertTo) || 0;
+
+    // both source and target must be configured
+    if (configuredSourceCcsid === 0 || configuredTargetCcsid === 0) {
+      return noConversion;
+    }
+
+    // source CCSID must match configured value
+    if (sourceCcsid != configuredSourceCcsid) {
+      return noConversion;
+    }
+
+    return { requiresConversion: true, targetCcsid: configuredTargetCcsid };
+  }
 }

src/api/configuration/config/ConnectionManager.ts

@@ -44,6 +44,9 @@ function initialize(parameters: Partial<ConnectionConfig>): ConnectionConfig {
     secureSQL: (parameters.secureSQL === true),
     keepActionSpooledFiles: (parameters.keepActionSpooledFiles === true),
     mapepireJavaVersion: (parameters.mapepireJavaVersion || "default"),
+    ccsidConversionEnabled: (parameters.ccsidConversionEnabled === true),
+    ccsidConvertFrom: parameters.ccsidConvertFrom || ``,
+    ccsidConvertTo: parameters.ccsidConvertTo || ``,
   }
 }
 

src/api/configuration/config/types.ts

@@ -16,6 +16,9 @@ export interface ConnectionConfig extends ConnectionProfile {
   hideCompileErrors: string[];
   enableSourceDates: boolean;
   sourceDateGutter: boolean;
+  ccsidConversionEnabled: boolean;
+  ccsidConvertFrom: string;
+  ccsidConvertTo: string;
   encodingFor5250: string;
   terminalFor5250: string;
   setDeviceNameFor5250: boolean;