Convert input handling based on CCSID scenarios

src/api/CompileTools.ts

@@ -147,7 +147,7 @@ export namespace CompileTools {
               command: [
                 ...options.noLibList? [] : buildLiblistCommands(connection, ileSetup),
                 ...commands.map(command =>
-                  `${`system "${IBMi.escapeForShell(command)}"`}`,
+                  `${`system "${IBMi.escapeForShell(connection.sysNameInAmerican(command, connection.systemCommandRequiresTranslation))}"`}`,
                 )
               ].join(` && `),
               directory: cwd,

src/api/IBMi.ts

@@ -19,6 +19,12 @@ import { EventEmitter } from 'stream';
 import { ConnectionConfig } from './configuration/config/types';
 import { EditorPath } from '../typings';
 
+export interface VariantInfo {
+  american: string,
+  local: string,
+  qsysNameRegex?: RegExp
+};
+
 export interface MemberParts extends IBMiMember {
   basename: string
 }
@@ -124,11 +130,7 @@ export default class IBMi {
 
   remoteFeatures: { [name: string]: string | undefined };
 
-  variantChars: {
-    american: string,
-    local: string,
-    qsysNameRegex?: RegExp
-  };
+  variantChars: VariantInfo;
 
   shell?: string;
 
@@ -160,16 +162,37 @@ export default class IBMi {
   }
 
   /**
-   * Determines if the client should do variant translation.
-   * False when cqsh should be used.
-   * True when cqsh is not available and the job CCSID is not the same as the SSHD CCSID.
+   * Determines if the client should do variant translation for SQL statements.
+   * The SQL runner (ZDFMDB2) translation is always based on QCCSID, and never on the user's CCSID.
+   */
+  private get sqlRunnerRequiresTranslation() {
+    if (this.canUseCqsh && this.qccsid === this.sshdCcsid) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Determine if QSYS paths need to be translated to use system variants
+   */
+  get qsysPosixPathsRequireTranslation() {
+    if (this.canUseCqsh && this.getCcsid() === this.qccsid && this.qccsid !== IBMi.CCSID_NOCONVERSION) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Determine if the system command requires translation
    */
-  get requiresTranslation() {
+  get systemCommandRequiresTranslation() {
     if (this.canUseCqsh) {
       return false;
-    } else {
-      return this.getCcsid() !== this.sshdCcsid;
     }
+
+    return true;
   }
 
   get dangerousVariants() {
@@ -851,17 +874,18 @@ export default class IBMi {
           }
         }
 
-        else {
-          // If cqsh is not available, then we need to check the SSHD CCSID
-          this.sshdCcsid = await this.content.getSshCcsid();
-          if (this.sshdCcsid === this.getCcsid()) {
-            // If the SSHD CCSID matches the job CCSID (not the user profile!), then we're good.
-            // This means we can use regular qsh without worrying about translation because the SSHD and job CCSID match.
-            userCcsidNeedsFixing = false;
-          } else {
-            // If the SSHD CCSID does not match the job CCSID, then we need to warn the user
-            sshdCcsidMismatch = true;
-          }
+        callbacks.progress({
+          message: `Checking SSHD CCSID.`
+        });
+
+        this.sshdCcsid = await this.content.getSshCcsid();
+        if (this.sshdCcsid === this.getCcsid()) {
+          // If the SSHD CCSID matches the job CCSID (not the user profile!), then we're good.
+          // This means we can use regular qsh without worrying about translation because the SSHD and job CCSID match.
+          userCcsidNeedsFixing = false;
+        } else {
+          // If the SSHD CCSID does not match the job CCSID, then we need to warn the user
+          sshdCcsidMismatch = true;
         }
 
         if (userCcsidNeedsFixing) {
@@ -1069,11 +1093,6 @@ export default class IBMi {
       qshExecutable = this.getComponent<CustomQSh>(CustomQSh.ID)!.installPath;
     }
 
-    if (this.requiresTranslation) {
-      options.stdin = this.sysNameInAmerican(options.stdin);
-      options.directory = options.directory ? this.sysNameInAmerican(options.directory) : undefined;
-    }
-
     return this.sendCommand({
       ...options,
       command: `${IBMi.locale} ${qshExecutable}`
@@ -1240,17 +1259,21 @@ export default class IBMi {
    * @param {string} string
    * @returns {string} result
    */
-  sysNameInAmerican(string: string) {
-    const fromChars = this.variantChars.local;
-    const toChars = this.variantChars.american;
+  sysNameInAmerican(string: string, enabled: boolean = true) {
+    if (enabled) {
+      const fromChars = this.variantChars.local;
+      const toChars = this.variantChars.american;
 
-    let result = string;
+      let result = string;
 
-    for (let i = 0; i < fromChars.length; i++) {
-      result = result.replace(new RegExp(`[${fromChars[i]}]`, `g`), toChars[i]);
-    };
+      for (let i = 0; i < fromChars.length; i++) {
+        result = result.replace(new RegExp(`[${fromChars[i]}]`, `g`), toChars[i]);
+      };
+
+      return result;
+    }
 
-    return result
+    return string;
   }
 
   getLastDownloadLocation() {
@@ -1346,7 +1369,7 @@ export default class IBMi {
         command = `${IBMi.locale} ${customQsh.installPath} -c "system \\"call QSYS/QZDFMDB2 PARM('-d' '-i' '-t')\\""`;
       }
 
-      if (this.requiresTranslation) {
+      if (this.sqlRunnerRequiresTranslation) {
         // If we can't fix the input, then we can attempt to convert ourselves and then use the CSV.
         input = this.sysNameInAmerican(input);
         useCsv = true;

src/api/IBMiContent.ts

@@ -167,7 +167,7 @@ export default class IBMiContent {
     const member = this.ibmi.upperCaseName(smallSignature ? sourceFileOrMember : String(memberOrLocalPath));
 
     const asp = await this.ibmi.lookupLibraryIAsp(library);
-    const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
+    const path = Tools.qualifyPath(library, sourceFile, member, asp);
     const tempRmt = this.getTempRemote(path);
     let retry = false;
     while (true) {
@@ -255,7 +255,7 @@ export default class IBMiContent {
 
     try {
       await writeFileAsync(tmpobj, content || memberOrContent, `utf8`);
-      const path = Tools.qualifyPath(library, sourceFile, member, asp, true);
+      const path = Tools.qualifyPath(library, sourceFile, member, asp);
       const tempRmt = this.getTempRemote(path);
       await client.putFile(tmpobj, tempRmt);
 
@@ -871,27 +871,31 @@ export default class IBMiContent {
   }
 
   async memberResolve(member: string, files: QsysPath[]): Promise<IBMiMember | undefined> {
-    const inAmerican = (s: string) => { return this.ibmi.sysNameInAmerican(s) };
-    const inLocal = (s: string) => { return this.ibmi.sysNameInLocal(s) };
+    const localVariants = this.ibmi.variantChars;
+    const forSystem = (s: string) => { return this.ibmi.sysNameInAmerican(s, this.ibmi.qsysPosixPathsRequireTranslation) };
+    const fromSystem = (s: string) => { return this.ibmi.qsysPosixPathsRequireTranslation ? this.ibmi.sysNameInLocal(s) : s };
 
     // Escape names for shell
     const pathList = files
       .map(file => {
         const asp = file.asp || this.ibmi.getCurrentIAspName();
         if (asp && asp.length > 0) {
           return [
-            Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), asp, true),
-            Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), undefined, true)
+            Tools.qualifyPath(forSystem(file.library), forSystem(file.name), forSystem(member), asp, localVariants),
+            Tools.qualifyPath(fromSystem(file.library), fromSystem(file.name), fromSystem(member), undefined, localVariants)
           ].join(` `);
         } else {
-          return Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), undefined, true);
+          return Tools.qualifyPath(forSystem(file.library), forSystem(file.name), forSystem(member), undefined, localVariants);
         }
       })
+      .map(
+        path => IBMi.escapeForShell(path)
+      )
       .join(` `)
       .toUpperCase();
 
     const command = `for f in ${pathList}; do if [ -f $f ]; then echo $f; break; fi; done`;
-    const result = await this.ibmi.sendCommand({
+    const result = await this.ibmi.sendQsh({
       command,
     });
 
@@ -900,7 +904,7 @@ export default class IBMiContent {
 
       if (firstMost) {
         try {
-          const simplePath = inLocal(Tools.unqualifyPath(firstMost));
+          const simplePath = fromSystem(Tools.unqualifyPath(firstMost));
 
           // This can error if the path format is wrong for some reason.
           // Not that this would ever happen, but better to be safe than sorry
@@ -915,7 +919,10 @@ export default class IBMiContent {
   }
 
   async objectResolve(object: string, libraries: string[]): Promise<string | undefined> {
-    const command = `for f in ${libraries.map(lib => `/QSYS.LIB/${this.ibmi.sysNameInAmerican(lib)}.LIB/${this.ibmi.sysNameInAmerican(object)}.*`).join(` `)}; do if [ -f $f ] || [ -d $f ]; then echo $f; break; fi; done`;
+    const forSystem = (s: string) => { return Tools.escapePath(this.ibmi.sysNameInAmerican(s, this.ibmi.qsysPosixPathsRequireTranslation)) };
+    const fromSystem = (s: string) => { return this.ibmi.qsysPosixPathsRequireTranslation ? this.ibmi.sysNameInLocal(s) : s };
+
+    const command = `for f in ${libraries.map(lib => `/QSYS.LIB/${forSystem(lib)}.LIB/${forSystem(object)}.*`).join(` `)}; do if [ -f $f ] || [ -d $f ]; then echo $f; break; fi; done`;
 
     const result = await this.ibmi.sendCommand({
       command,
@@ -925,7 +932,7 @@ export default class IBMiContent {
       const firstMost = result.stdout;
 
       if (firstMost) {
-        const lib = this.ibmi.sysNameInLocal(Tools.unqualifyPath(firstMost));
+        const lib = fromSystem(Tools.unqualifyPath(firstMost));
 
         return lib.split('/')[1];
       }
@@ -1035,11 +1042,11 @@ export default class IBMiContent {
 
     if (assumeMember) {
       // If it's an object, we assume it's a member, therefore let's let qsh handle it (better for variants)
-      localPath.asp = localPath.asp ? this.ibmi.sysNameInAmerican(localPath.asp) : undefined;
-      localPath.library = this.ibmi.sysNameInAmerican(localPath.library);
-      localPath.name = this.ibmi.sysNameInAmerican(localPath.name);
-      localPath.member = localPath.member ? this.ibmi.sysNameInAmerican(localPath.member) : undefined;
-      target = Tools.qualifyPath(localPath.library, localPath.name, localPath.member || '', localPath.asp || '', true);
+      target = Tools.qualifyPath(localPath.library, localPath.name, localPath.member || '', localPath.asp || '', this.ibmi.variantChars);
+
+      if (this.ibmi.qsysPosixPathsRequireTranslation) {
+        target = this.ibmi.sysNameInAmerican(target);
+      }
     } else {
       target = localPath;
     }
@@ -1067,11 +1074,18 @@ export default class IBMiContent {
   }
 
   async countMembers(path: QsysPath) {
-    return this.countFiles(this.ibmi.sysNameInAmerican(Tools.qualifyPath(path.library, path.name, undefined, path.asp)))
+    return this.countFiles(this.ibmi.sysNameInAmerican(
+      Tools.qualifyPath(path.library, path.name, undefined, path.asp),
+      this.ibmi.qsysPosixPathsRequireTranslation
+    ), true);
   }
 
-  async countFiles(directory: string) {
-    return Number((await this.ibmi.sendCommand({ command: `cd "${directory}" && (ls | wc -l)` })).stdout.trim());
+  async countFiles(directory: string, isQsys?: boolean) {
+    if (isQsys) {
+      return Number((await this.ibmi.sendQsh({ command: `cd "${IBMi.escapeForShell(directory)}" && (ls | wc -l)` })).stdout.trim());
+    } else {
+      return Number((await this.ibmi.sendCommand({ command: `cd "${IBMi.escapeForShell(directory)}" && (ls | wc -l)` })).stdout.trim());
+    }
   }
 
 

src/api/Search.ts

@@ -14,7 +14,7 @@ export namespace Search {
       let memberFilter: string|undefined;
 
       if (typeof members === `string`) {
-        memberFilter = connection.sysNameInAmerican(`${members}.MBR`);
+        memberFilter = connection.sysNameInAmerican(`${members}.MBR`, connection.qsysPosixPathsRequireTranslation);
       } else
       if (Array.isArray(members)) {
         if (members.length > connection.maximumArgsLength) {
@@ -31,7 +31,7 @@ export namespace Search {
       // Then search the members
       const result = await connection.sendQsh({
         command: `/usr/bin/grep -inHR -F "${sanitizeSearchTerm(searchTerm)}" ${memberFilter}`,
-        directory: connection.sysNameInAmerican(`${asp ? `/${asp}` : ``}/QSYS.LIB/${library}.LIB/${sourceFile}.FILE`)
+        directory: connection.sysNameInAmerican(`${asp ? `/${asp}` : ``}/QSYS.LIB/${library}.LIB/${sourceFile}.FILE`, connection.qsysPosixPathsRequireTranslation)
       });
 
       if (!result.stderr) {

src/api/Tools.ts

@@ -3,6 +3,7 @@ import os from "os";
 import path from "path";
 import { IBMiMessage, IBMiMessages, QsysPath } from './types';
 import { EditorPath } from "../typings";
+import { VariantInfo } from "./IBMi";
 
 export namespace Tools {
   export class SqlError extends Error {
@@ -176,17 +177,17 @@ export namespace Tools {
    * @param member Optional
    * @param iasp Optional: an iASP name
    */
-  export function qualifyPath(library: string, object: string, member?: string, iasp?: string, noEscape?: boolean) {
-    [library, object] = Tools.sanitizeObjNamesForPase([library, object]);
-    member = member ? Tools.sanitizeObjNamesForPase([member])[0] : undefined;
-    iasp = iasp ? Tools.sanitizeObjNamesForPase([iasp])[0] : undefined;
+  export function qualifyPath(library: string, object: string, member?: string, iasp?: string, localVariants?: VariantInfo) {
+    [library, object] = Tools.sanitizeObjNamesForPase([library, object], localVariants);
+    member = member ? Tools.sanitizeObjNamesForPase([member], localVariants)[0] : undefined;
+    iasp = iasp ? Tools.sanitizeObjNamesForPase([iasp], localVariants)[0] : undefined;
 
     const libraryPath = library === `QSYS` ? `QSYS.LIB` : `QSYS.LIB/${library}.LIB`;
     const filePath = object ? `${object}.FILE` : '';
     const memberPath = member ? `/${member}.MBR` : '';
     const fullPath = `${libraryPath}/${filePath}${memberPath}`;
 
-    const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${noEscape ? fullPath : Tools.escapePath(fullPath)}`;
+    const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${fullPath}`;
     return result;
   }
 
@@ -236,11 +237,13 @@ export namespace Tools {
     return text.charAt(0).toUpperCase() + text.slice(1);
   }
 
-  export function sanitizeObjNamesForPase(libraries: string[]): string[] {
+  export function sanitizeObjNamesForPase(libraries: string[], localVariants?: VariantInfo): string[] {
+    const checkChar = localVariants ? localVariants.local[0] : `"`;
     return libraries
       .map(library => {
-        // Quote libraries starting with #
-        return library.startsWith(`#`) ? `"${library}"` : library;
+        const first = library[0];
+
+        return first === checkChar ? `"${library}"` : library;
       });
   }
 

src/api/tests/setup.ts

@@ -10,9 +10,22 @@ export async function setup(project: TestProject) {
     console.log(`Testing connection before tests run since configs do not exist.`);
 
     const conn = await newConnection();
-    disposeConnection(conn);
 
     console.log(`Testing connection complete. Configs written.`);
     console.log(``);
+
+    console.table({
+      "QCCSID": conn.getCcsids().qccsid,
+      "runtimeCcsid": conn.getCcsids().runtimeCcsid,
+      "userDefaultCCSID": conn.getCcsids().userDefaultCCSID,
+      "sshdCcsid": conn.getCcsids().sshdCcsid,
+      "canUseCqsh": conn.canUseCqsh,
+      "americanVariants": conn.variantChars.american,
+      "localVariants": conn.variantChars.local,
+    });
+
+    console.log(``);
+
+    disposeConnection(conn);
   }
 }