diff --git a/src/api/CompileTools.ts b/src/api/CompileTools.ts index 4c1f0c3bc..866abf344 100644 --- a/src/api/CompileTools.ts +++ b/src/api/CompileTools.ts @@ -147,7 +147,7 @@ export namespace CompileTools { command: [ ...options.noLibList? [] : buildLiblistCommands(connection, ileSetup), ...commands.map(command => - `${`system "${IBMi.escapeForShell(command)}"`}`, + `${`system "${IBMi.escapeForShell(connection.sysNameInAmerican(command, connection.systemCommandRequiresTranslation))}"`}`, ) ].join(` && `), directory: cwd, diff --git a/src/api/IBMi.ts b/src/api/IBMi.ts index f0f0877df..70a2b4e5e 100644 --- a/src/api/IBMi.ts +++ b/src/api/IBMi.ts @@ -19,6 +19,12 @@ import { EventEmitter } from 'stream'; import { ConnectionConfig } from './configuration/config/types'; import { EditorPath } from '../typings'; +export interface VariantInfo { + american: string, + local: string, + qsysNameRegex?: RegExp +}; + export interface MemberParts extends IBMiMember { basename: string } @@ -124,11 +130,7 @@ export default class IBMi { remoteFeatures: { [name: string]: string | undefined }; - variantChars: { - american: string, - local: string, - qsysNameRegex?: RegExp - }; + variantChars: VariantInfo; shell?: string; @@ -160,16 +162,37 @@ export default class IBMi { } /** - * Determines if the client should do variant translation. - * False when cqsh should be used. - * True when cqsh is not available and the job CCSID is not the same as the SSHD CCSID. + * Determines if the client should do variant translation for SQL statements. + * The SQL runner (ZDFMDB2) translation is always based on QCCSID, and never on the user's CCSID. + */ + private get sqlRunnerRequiresTranslation() { + if (this.canUseCqsh && this.qccsid === this.sshdCcsid) { + return false; + } + + return true; + } + + /** + * Determine if QSYS paths need to be translated to use system variants + */ + get qsysPosixPathsRequireTranslation() { + if (this.canUseCqsh && this.getCcsid() === this.qccsid && this.qccsid !== IBMi.CCSID_NOCONVERSION) { + return false; + } + + return true; + } + + /** + * Determine if the system command requires translation */ - get requiresTranslation() { + get systemCommandRequiresTranslation() { if (this.canUseCqsh) { return false; - } else { - return this.getCcsid() !== this.sshdCcsid; } + + return true; } get dangerousVariants() { @@ -851,17 +874,18 @@ export default class IBMi { } } - else { - // If cqsh is not available, then we need to check the SSHD CCSID - this.sshdCcsid = await this.content.getSshCcsid(); - if (this.sshdCcsid === this.getCcsid()) { - // If the SSHD CCSID matches the job CCSID (not the user profile!), then we're good. - // This means we can use regular qsh without worrying about translation because the SSHD and job CCSID match. - userCcsidNeedsFixing = false; - } else { - // If the SSHD CCSID does not match the job CCSID, then we need to warn the user - sshdCcsidMismatch = true; - } + callbacks.progress({ + message: `Checking SSHD CCSID.` + }); + + this.sshdCcsid = await this.content.getSshCcsid(); + if (this.sshdCcsid === this.getCcsid()) { + // If the SSHD CCSID matches the job CCSID (not the user profile!), then we're good. + // This means we can use regular qsh without worrying about translation because the SSHD and job CCSID match. + userCcsidNeedsFixing = false; + } else { + // If the SSHD CCSID does not match the job CCSID, then we need to warn the user + sshdCcsidMismatch = true; } if (userCcsidNeedsFixing) { @@ -1069,11 +1093,6 @@ export default class IBMi { qshExecutable = this.getComponent(CustomQSh.ID)!.installPath; } - if (this.requiresTranslation) { - options.stdin = this.sysNameInAmerican(options.stdin); - options.directory = options.directory ? this.sysNameInAmerican(options.directory) : undefined; - } - return this.sendCommand({ ...options, command: `${IBMi.locale} ${qshExecutable}` @@ -1240,17 +1259,21 @@ export default class IBMi { * @param {string} string * @returns {string} result */ - sysNameInAmerican(string: string) { - const fromChars = this.variantChars.local; - const toChars = this.variantChars.american; + sysNameInAmerican(string: string, enabled: boolean = true) { + if (enabled) { + const fromChars = this.variantChars.local; + const toChars = this.variantChars.american; - let result = string; + let result = string; - for (let i = 0; i < fromChars.length; i++) { - result = result.replace(new RegExp(`[${fromChars[i]}]`, `g`), toChars[i]); - }; + for (let i = 0; i < fromChars.length; i++) { + result = result.replace(new RegExp(`[${fromChars[i]}]`, `g`), toChars[i]); + }; + + return result; + } - return result + return string; } getLastDownloadLocation() { @@ -1346,7 +1369,7 @@ export default class IBMi { command = `${IBMi.locale} ${customQsh.installPath} -c "system \\"call QSYS/QZDFMDB2 PARM('-d' '-i' '-t')\\""`; } - if (this.requiresTranslation) { + if (this.sqlRunnerRequiresTranslation) { // If we can't fix the input, then we can attempt to convert ourselves and then use the CSV. input = this.sysNameInAmerican(input); useCsv = true; diff --git a/src/api/IBMiContent.ts b/src/api/IBMiContent.ts index 1dc5f02bf..3e74df6b5 100644 --- a/src/api/IBMiContent.ts +++ b/src/api/IBMiContent.ts @@ -167,7 +167,7 @@ export default class IBMiContent { const member = this.ibmi.upperCaseName(smallSignature ? sourceFileOrMember : String(memberOrLocalPath)); const asp = await this.ibmi.lookupLibraryIAsp(library); - const path = Tools.qualifyPath(library, sourceFile, member, asp, true); + const path = Tools.qualifyPath(library, sourceFile, member, asp); const tempRmt = this.getTempRemote(path); let retry = false; while (true) { @@ -255,7 +255,7 @@ export default class IBMiContent { try { await writeFileAsync(tmpobj, content || memberOrContent, `utf8`); - const path = Tools.qualifyPath(library, sourceFile, member, asp, true); + const path = Tools.qualifyPath(library, sourceFile, member, asp); const tempRmt = this.getTempRemote(path); await client.putFile(tmpobj, tempRmt); @@ -871,8 +871,9 @@ export default class IBMiContent { } async memberResolve(member: string, files: QsysPath[]): Promise { - const inAmerican = (s: string) => { return this.ibmi.sysNameInAmerican(s) }; - const inLocal = (s: string) => { return this.ibmi.sysNameInLocal(s) }; + const localVariants = this.ibmi.variantChars; + const forSystem = (s: string) => { return this.ibmi.sysNameInAmerican(s, this.ibmi.qsysPosixPathsRequireTranslation) }; + const fromSystem = (s: string) => { return this.ibmi.qsysPosixPathsRequireTranslation ? this.ibmi.sysNameInLocal(s) : s }; // Escape names for shell const pathList = files @@ -880,18 +881,21 @@ export default class IBMiContent { const asp = file.asp || this.ibmi.getCurrentIAspName(); if (asp && asp.length > 0) { return [ - Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), asp, true), - Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), undefined, true) + Tools.qualifyPath(forSystem(file.library), forSystem(file.name), forSystem(member), asp, localVariants), + Tools.qualifyPath(fromSystem(file.library), fromSystem(file.name), fromSystem(member), undefined, localVariants) ].join(` `); } else { - return Tools.qualifyPath(inAmerican(file.library), inAmerican(file.name), inAmerican(member), undefined, true); + return Tools.qualifyPath(forSystem(file.library), forSystem(file.name), forSystem(member), undefined, localVariants); } }) + .map( + path => IBMi.escapeForShell(path) + ) .join(` `) .toUpperCase(); const command = `for f in ${pathList}; do if [ -f $f ]; then echo $f; break; fi; done`; - const result = await this.ibmi.sendCommand({ + const result = await this.ibmi.sendQsh({ command, }); @@ -900,7 +904,7 @@ export default class IBMiContent { if (firstMost) { try { - const simplePath = inLocal(Tools.unqualifyPath(firstMost)); + const simplePath = fromSystem(Tools.unqualifyPath(firstMost)); // This can error if the path format is wrong for some reason. // Not that this would ever happen, but better to be safe than sorry @@ -915,7 +919,10 @@ export default class IBMiContent { } async objectResolve(object: string, libraries: string[]): Promise { - const command = `for f in ${libraries.map(lib => `/QSYS.LIB/${this.ibmi.sysNameInAmerican(lib)}.LIB/${this.ibmi.sysNameInAmerican(object)}.*`).join(` `)}; do if [ -f $f ] || [ -d $f ]; then echo $f; break; fi; done`; + const forSystem = (s: string) => { return Tools.escapePath(this.ibmi.sysNameInAmerican(s, this.ibmi.qsysPosixPathsRequireTranslation)) }; + const fromSystem = (s: string) => { return this.ibmi.qsysPosixPathsRequireTranslation ? this.ibmi.sysNameInLocal(s) : s }; + + const command = `for f in ${libraries.map(lib => `/QSYS.LIB/${forSystem(lib)}.LIB/${forSystem(object)}.*`).join(` `)}; do if [ -f $f ] || [ -d $f ]; then echo $f; break; fi; done`; const result = await this.ibmi.sendCommand({ command, @@ -925,7 +932,7 @@ export default class IBMiContent { const firstMost = result.stdout; if (firstMost) { - const lib = this.ibmi.sysNameInLocal(Tools.unqualifyPath(firstMost)); + const lib = fromSystem(Tools.unqualifyPath(firstMost)); return lib.split('/')[1]; } @@ -1035,11 +1042,11 @@ export default class IBMiContent { if (assumeMember) { // If it's an object, we assume it's a member, therefore let's let qsh handle it (better for variants) - localPath.asp = localPath.asp ? this.ibmi.sysNameInAmerican(localPath.asp) : undefined; - localPath.library = this.ibmi.sysNameInAmerican(localPath.library); - localPath.name = this.ibmi.sysNameInAmerican(localPath.name); - localPath.member = localPath.member ? this.ibmi.sysNameInAmerican(localPath.member) : undefined; - target = Tools.qualifyPath(localPath.library, localPath.name, localPath.member || '', localPath.asp || '', true); + target = Tools.qualifyPath(localPath.library, localPath.name, localPath.member || '', localPath.asp || '', this.ibmi.variantChars); + + if (this.ibmi.qsysPosixPathsRequireTranslation) { + target = this.ibmi.sysNameInAmerican(target); + } } else { target = localPath; } @@ -1067,11 +1074,18 @@ export default class IBMiContent { } async countMembers(path: QsysPath) { - return this.countFiles(this.ibmi.sysNameInAmerican(Tools.qualifyPath(path.library, path.name, undefined, path.asp))) + return this.countFiles(this.ibmi.sysNameInAmerican( + Tools.qualifyPath(path.library, path.name, undefined, path.asp), + this.ibmi.qsysPosixPathsRequireTranslation + ), true); } - async countFiles(directory: string) { - return Number((await this.ibmi.sendCommand({ command: `cd "${directory}" && (ls | wc -l)` })).stdout.trim()); + async countFiles(directory: string, isQsys?: boolean) { + if (isQsys) { + return Number((await this.ibmi.sendQsh({ command: `cd "${IBMi.escapeForShell(directory)}" && (ls | wc -l)` })).stdout.trim()); + } else { + return Number((await this.ibmi.sendCommand({ command: `cd "${IBMi.escapeForShell(directory)}" && (ls | wc -l)` })).stdout.trim()); + } } diff --git a/src/api/Search.ts b/src/api/Search.ts index 68987bf83..c06fd28b3 100644 --- a/src/api/Search.ts +++ b/src/api/Search.ts @@ -14,7 +14,7 @@ export namespace Search { let memberFilter: string|undefined; if (typeof members === `string`) { - memberFilter = connection.sysNameInAmerican(`${members}.MBR`); + memberFilter = connection.sysNameInAmerican(`${members}.MBR`, connection.qsysPosixPathsRequireTranslation); } else if (Array.isArray(members)) { if (members.length > connection.maximumArgsLength) { @@ -31,7 +31,7 @@ export namespace Search { // Then search the members const result = await connection.sendQsh({ command: `/usr/bin/grep -inHR -F "${sanitizeSearchTerm(searchTerm)}" ${memberFilter}`, - directory: connection.sysNameInAmerican(`${asp ? `/${asp}` : ``}/QSYS.LIB/${library}.LIB/${sourceFile}.FILE`) + directory: connection.sysNameInAmerican(`${asp ? `/${asp}` : ``}/QSYS.LIB/${library}.LIB/${sourceFile}.FILE`, connection.qsysPosixPathsRequireTranslation) }); if (!result.stderr) { diff --git a/src/api/Tools.ts b/src/api/Tools.ts index 3df788656..406540522 100644 --- a/src/api/Tools.ts +++ b/src/api/Tools.ts @@ -3,6 +3,7 @@ import os from "os"; import path from "path"; import { IBMiMessage, IBMiMessages, QsysPath } from './types'; import { EditorPath } from "../typings"; +import { VariantInfo } from "./IBMi"; export namespace Tools { export class SqlError extends Error { @@ -176,17 +177,17 @@ export namespace Tools { * @param member Optional * @param iasp Optional: an iASP name */ - export function qualifyPath(library: string, object: string, member?: string, iasp?: string, noEscape?: boolean) { - [library, object] = Tools.sanitizeObjNamesForPase([library, object]); - member = member ? Tools.sanitizeObjNamesForPase([member])[0] : undefined; - iasp = iasp ? Tools.sanitizeObjNamesForPase([iasp])[0] : undefined; + export function qualifyPath(library: string, object: string, member?: string, iasp?: string, localVariants?: VariantInfo) { + [library, object] = Tools.sanitizeObjNamesForPase([library, object], localVariants); + member = member ? Tools.sanitizeObjNamesForPase([member], localVariants)[0] : undefined; + iasp = iasp ? Tools.sanitizeObjNamesForPase([iasp], localVariants)[0] : undefined; const libraryPath = library === `QSYS` ? `QSYS.LIB` : `QSYS.LIB/${library}.LIB`; const filePath = object ? `${object}.FILE` : ''; const memberPath = member ? `/${member}.MBR` : ''; const fullPath = `${libraryPath}/${filePath}${memberPath}`; - const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${noEscape ? fullPath : Tools.escapePath(fullPath)}`; + const result = (iasp && iasp.length > 0 ? `/${iasp}` : ``) + `/${fullPath}`; return result; } @@ -236,11 +237,13 @@ export namespace Tools { return text.charAt(0).toUpperCase() + text.slice(1); } - export function sanitizeObjNamesForPase(libraries: string[]): string[] { + export function sanitizeObjNamesForPase(libraries: string[], localVariants?: VariantInfo): string[] { + const checkChar = localVariants ? localVariants.local[0] : `"`; return libraries .map(library => { - // Quote libraries starting with # - return library.startsWith(`#`) ? `"${library}"` : library; + const first = library[0]; + + return first === checkChar ? `"${library}"` : library; }); } diff --git a/src/api/tests/setup.ts b/src/api/tests/setup.ts index 43baac0ce..094d775b9 100644 --- a/src/api/tests/setup.ts +++ b/src/api/tests/setup.ts @@ -10,9 +10,22 @@ export async function setup(project: TestProject) { console.log(`Testing connection before tests run since configs do not exist.`); const conn = await newConnection(); - disposeConnection(conn); console.log(`Testing connection complete. Configs written.`); console.log(``); + + console.table({ + "QCCSID": conn.getCcsids().qccsid, + "runtimeCcsid": conn.getCcsids().runtimeCcsid, + "userDefaultCCSID": conn.getCcsids().userDefaultCCSID, + "sshdCcsid": conn.getCcsids().sshdCcsid, + "canUseCqsh": conn.canUseCqsh, + "americanVariants": conn.variantChars.american, + "localVariants": conn.variantChars.local, + }); + + console.log(``); + + disposeConnection(conn); } } \ No newline at end of file diff --git a/src/api/tests/suites/encoding.test.ts b/src/api/tests/suites/encoding.test.ts index f6fe4c3f2..662504317 100644 --- a/src/api/tests/suites/encoding.test.ts +++ b/src/api/tests/suites/encoding.test.ts @@ -176,10 +176,13 @@ describe('Encoding tests', { concurrent: true }, () => { const memberContentA = await content?.downloadMemberContent(tempLib, tempSPF, tempMbr); expect(memberContentA).toBe(baseContent); + + const resolved = await content?.memberResolve(tempMbr, [{name: tempSPF, library: `QSYS`}, {name: tempSPF, library: `QSY2`}, {name: tempSPF, library: tempLib}]); + expect(resolved).toBeTruthy(); }); }); - it('Listing objects with variants', { timeout: 15000 }, async () => { + it('Listing objects with variants', { timeout: 40000 }, async () => { const content = connection.getContent(); if (connection && content) { const tempLib = connection.config?.tempLibrary!; @@ -188,7 +191,7 @@ describe('Encoding tests', { concurrent: true }, () => { let library = `TESTLIB${connection.variantChars.local}`; let skipLibrary = false; const sourceFile = `${connection.variantChars.local}TESTFIL`; - const dataArea = `TSTDTA${connection.variantChars.local}`; + const dataArea = `${connection.variantChars.local}TSTDTA`; const members: string[] = []; for (let i = 0; i < 5; i++) { @@ -236,7 +239,7 @@ describe('Encoding tests', { concurrent: true }, () => { }; const nameFilter = await content.getObjectList({ library, types: ["*ALL"], object: `${connection.variantChars.local[0]}*` }); - expect(nameFilter.length).toBe(1); + expect(nameFilter.length).toBe(2); expect(nameFilter.some(obj => obj.library === library && obj.type === `*FILE` && obj.name === sourceFile)).toBeTruthy(); const objectList = await content.getObjectList({ library, types: ["*ALL"] }); @@ -258,6 +261,9 @@ describe('Encoding tests', { concurrent: true }, () => { const [expectedSourceFile] = await content.getObjectList({ library, object: sourceFile, types: ["*SRCPF"] }); checkFile(expectedSourceFile); + + const resolvedObject = await content.objectResolve(dataArea, [`QSYS`, `QSYS2`, library]); + expect(resolvedObject).toBeTruthy(); } }); @@ -291,7 +297,7 @@ describe('Encoding tests', { concurrent: true }, () => { } }); - it('Variant character in source names and commands', { timeout: 45000 }, async () => { + it('Variant character in source names and commands', { timeout: 55000 }, async () => { const config = connection.getConfig(); const ccsidData = connection.getCcsids()!; const tempLib = config.tempLibrary; @@ -337,10 +343,18 @@ describe('Encoding tests', { concurrent: true }, () => { expect(files.some(f => f.name === connection.sysNameInAmerican(variantMember) + `.MBR`)).toBeTruthy(); expect(files.some(f => f.name === connection.sysNameInAmerican(testMember) + `.MBR`)).toBeTruthy(); - await connection.content.uploadMemberContent(tempLib, testFile, testMember, [`**free`, `dsply 'Hello world';`, ` `, ` `, `return;`].join(`\n`)); + const count = await connection.getContent().countMembers({library: tempLib, name: testFile}); + expect(count).toBe(2); + + const eol = `\r\n`; + const memberContents = [`**free`, `dsply 'Hello world';`, ``, ``, `return;`].join(eol); + await connection.getContent().uploadMemberContent(tempLib, testFile, testMember, memberContents); + + const downloadedContent = await connection.getContent().downloadMemberContent(tempLib, testFile, testMember); + + expect(downloadedContent).toBe(memberContents + eol); const compileResult = await connection.runCommand({ command: `CRTBNDRPG PGM(${tempLib}/${testMember}) SRCFILE(${tempLib}/${testFile}) SRCMBR(${testMember})`, noLibList: true }); - console.log(compileResult); expect(compileResult.code).toBe(0); if (compileResult.code === 0) {