Skip to content

Commit 1b6b60d

Browse files
jfeingold35jfeingold35
authored
Merge pull request #1651 from forcedotcom/d/W-17014756
NEW @W-17014756@ Adding functionality to smoke tests
2 parents 92883b0 + 010c10b commit 1b6b60d

File tree

10 files changed

+302
-11
lines changed

10 files changed

+302
-11
lines changed

‎.github/workflows/run-tests.yml‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,35 @@ jobs:
4646
path: |
4747
./reports
4848
./coverage
49+
50+
# The smoke tests should mimic a production setup as closely as possible. To that end, we'll spin up a tarball and
51+
# then upload it as an artifact, so the smoke test jobs can pull it down and install it.
52+
build-installable-tarball:
53+
runs-on: ubuntu-latest
54+
steps:
55+
# Obviously, we need to get the code first
56+
- uses: actions/checkout@v4
57+
with:
58+
ref: ${{ inputs.target-branch }}
59+
# Make sure we're on the right version of Node
60+
- uses: actions/setup-node@v4
61+
with:
62+
node-version: 'lts/*' # Always use Node LTS for building the tarball
63+
# Install/build dependencies
64+
- run: yarn
65+
- run: yarn build
66+
# Create the tarball
67+
- run: npm pack
68+
# Upload the tarball as an artifact
69+
- uses: actions/upload-artifact@v4
70+
with:
71+
name: smoke-test-tarball
72+
path: ./salesforce-plugin-code-analyzer-*.tgz
4973
# For now, the smoke tests are no-ops, so the job doesn't have to do anything exciting.
5074
# When smoke-tests stop being no-ops, we'll need a build-dependencies step to create a tarball artifact, and so forth.
5175
# We'll be able to use the v4 workflow as a template.
5276
smoke-tests:
77+
needs: build-installable-tarball
5378
strategy:
5479
# By default, if any job in a matrix fails, all other jobs are automatically cancelled. This makes the jobs run
5580
# to completion instead.
@@ -59,8 +84,47 @@ jobs:
5984
os: [{vm: ubuntu-latest, exe: .sh}, {vm: macos-latest, exe: .sh}, {vm: windows-latest, exe: .cmd}]
6085
runs-on: ${{ matrix.os.vm }}
6186
steps:
87+
# Check out the code
6288
- uses: actions/checkout@v4
6389
with:
6490
ref: ${{ inputs.target-branch }}
65-
- run: smoke-tests/smoke-test${{ matrix.os.exe }} sf
91+
# Set up Node and Java
92+
- uses: actions/setup-node@v4
93+
with:
94+
node-version: ${{ matrix.node.version }}
95+
- uses: actions/setup-java@v4
96+
with:
97+
distribution: 'temurin'
98+
java-version: '11'
99+
# Install SF CLI via NPM
100+
- run: npm install -g @salesforce/cli
101+
# Download and install the Tarball artifact
102+
- uses: actions/download-artifact@v4
103+
id: download
104+
with:
105+
name: smoke-test-tarball
106+
# Download the tarball to a subdirectory of HOME, to guarantee that it's somewhere the installation command
107+
# can see
108+
path: ~/downloads/tarball
109+
- name: Echo tarball download location
110+
run: echo ${{ steps.download.outputs.download-path }}
111+
- name: Install Tarball
112+
shell: bash
113+
run: |
114+
# We need to determine the Tarball's name first.
115+
TARBALL_NAME=$(ls ~/downloads/tarball | grep salesforce-plugin-code-analyzer-5\\.0\\.0-alpha\\.[0-9]*\\.tgz)
116+
# We need to determine the Tarball's location in an installable way.
117+
# Get the path to the download folder. Swap out backslashes for forward slashes to ensure Windows compatibility.
118+
RAW_TARBALL_PATH=`echo '${{ steps.download.outputs.download-path }}' | tr '\\' '/'`
119+
# If the path starts with "C:", that needs to be pulled off.
120+
ADJUSTED_TARBALL_PATH=`[[ $RAW_TARBALL_PATH = C* ]] && echo $RAW_TARBALL_PATH | cut -d':' -f 2 || echo $RAW_TARBALL_PATH`
121+
# Install the tarball, piping in a `y` to simulate agreeing to install an unsigned package. Use the URI of the file's full path.
122+
echo y | sf plugins install "file://${ADJUSTED_TARBALL_PATH}/${TARBALL_NAME}"
123+
- name: Run smoke tests
124+
run: smoke-tests/smoke-test${{ matrix.os.exe }} sf
125+
- uses: actions/upload-artifact@v4
126+
if: ${{always()}}
127+
with:
128+
name: smoke-test-results-${{ runner.os }}-node-${{ matrix.node.artifact }}
129+
path: smoke-test-results
66130

‎smoke-tests/smoke-test-generator.js‎

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
const fs = require('fs');
2+
const path = require('path');
23

34
console.log('Switching to smoke test directory');
45
process.chdir(__dirname);
@@ -13,7 +14,7 @@ function generateScriptHeader(isBash) {
1314

1415
// All scripts should start with the same boilerplate disclaimer.
1516
header += `${cmt} Auto-generated on ${new Date(Date.now()).toDateString()}
16-
${cmt} This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
17+
${cmt} This script performs a smoke test of the entire plugin by running a series of commands that collectively
1718
${cmt} hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
1819
${cmt} approximately stable.
1920
${cmt} DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
@@ -31,11 +32,87 @@ ${cmt} SCRIPT FROM THE PROJECT ROOT!\n`;
3132
}
3233

3334
function generateScriptBody(isBash, delim) {
35+
const exeName = isBash ? '$EXE_NAME' : '%EXE_NAME%';
36+
37+
3438
const commands = [
35-
// Log an explainer that since we're in the Alpha stage, we don't have a fully fleshed out smoke test.
36-
`echo "At this point in the alpha, the smoke tests are no-ops and it is fine."`
37-
]
39+
// Log a header.
40+
`echo "====== SETUP FOR SMOKE TESTS ======"`,
41+
// Delete the smoke test result directory if it already exists
42+
`echo "====== Delete a a pre-existing smoke-test-results directory ======"`,
43+
isBash ? 'rm -rf smoke-test-results' : 'if exist smoke-test-results rmdir /s /q smoke-test-results || exit /b 1',
44+
// Create the output directory
45+
`echo "====== Create smoke-test-results directory ======"`,
46+
isBash ? 'mkdir -p smoke-test-results' : 'if not exist smoke-test-results mkdir smoke-test-results || exit /b 1',
47+
`\n`,
48+
`echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"`,
49+
// Show the default configuration for all engines
50+
`echo "====== Show default config for all engines ======"`,
51+
`${exeName} code-analyzer config`,
52+
// Show the default configuration for only one engine
53+
`echo "====== Show default config for PMD only ======"`,
54+
`${exeName} code-analyzer config -r pmd`,
55+
// Write the default config for PMD to a file
56+
`echo "====== Write PMD's default config to a file ======"`,
57+
`${exeName} code-analyzer config -r pmd -f ${path.join('.', 'smoke-test-results', 'pmd-only-config.yml')}`,
58+
// Show the configuration from the config file we just created
59+
`echo "====== Show configuration from last step's output file ======"`,
60+
`${exeName} code-analyzer config -c ${path.join('.', 'smoke-test-results', 'pmd-only-config.yml')}`,
61+
`echo "====== Show configuration from pre-existing config file ======"`,
62+
`${exeName} code-analyzer config -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')}`,
63+
`\n`,
64+
`echo "====== SMOKE TESTS FOR RULES COMMAND ======"`,
65+
// List all rules
66+
`echo "====== List all rules ======"`,
67+
`${exeName} code-analyzer rules`,
68+
// List only rules from ESLint
69+
`echo "====== List ESLint rules only ======"`,
70+
`${exeName} code-analyzer rules -r eslint`,
71+
// List only rules from RetireJS
72+
`echo "====== List RetireJS rules only ======"`,
73+
`${exeName} code-analyzer rules -r retire-js`,
74+
// List only rules that are relevant to a provided workspace
75+
`echo "====== List rules relevant to apex-only workspace ======"`,
76+
`${exeName} code-analyzer rules -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-apex-files')}`,
77+
// List only rules that match nonsensical selector (i.e., list no rules)
78+
`echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"`,
79+
`${exeName} code-analyzer rules -r asdfasdfasdf`,
80+
// List one rule using a config with overrides
81+
`echo "====== List rule overridden in config file ======"`,
82+
`${exeName} code-analyzer rules -r no-unsafe-assignment -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')}`,
83+
'\n',
84+
`echo "====== SMOKE TESTS FOR RUN COMMAND ======"`,
85+
// Run all rules against a folder
86+
`echo "====== Run all rules against a folder ======"`,
87+
`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
88+
// Run all rules against a file
89+
`echo "====== Run all rules against a file ======"`,
90+
`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files', 'my-script.ts')}`,
91+
// Run all rules against a folder and write the output to some files
92+
`echo "====== Run all rules against a folder and write to outfiles ======"`,
93+
`${exeName} code-analyzer run -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-apex-files')} -f ${path.join('.', 'smoke-test-results', 'outfile.json')} -f ${path.join('.', 'smoke-test-results', 'outfile.html')}`,
94+
// Run a selection of rules against a folder
95+
`echo "====== Run a selection of rules against a folder ======"`,
96+
`${exeName} code-analyzer run -r regex -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
97+
// Run rules using a config file with overrides
98+
`echo "====== Run rules using a config file with overrides ======"`,
99+
`${exeName} code-analyzer run -c ${path.join('.', 'smoke-tests', 'test-data', 'config-files', 'existing-config.yml')} -w ${path.join('.', 'smoke-tests', 'test-data', 'workspace-with-mixed-files')}`,
100+
'\n',
101+
`echo "====== CONCLUSION ======"`,
102+
`echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"`
103+
];
104+
105+
// In a cmd script, you need to prepend plugin commands with "call" in order to make sure that the script continues,
106+
// and you need to postfix it with another snippet to make it actually exist when an error is encountered.
107+
if (!isBash) {
108+
for (let i = 0; i < commands.length; i++) {
109+
if (commands[i].startsWith(exeName)) {
110+
commands[i] = `call ${commands[i]} || exit /b 1`
111+
}
112+
}
113+
}
38114

115+
// Combine the commands and return the script body.
39116
return commands.join('\n');
40117
}
41118

‎smoke-tests/smoke-test.cmd‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,58 @@
11
@echo off
2-
REM Auto-generated on Tue Aug 27 2024
3-
REM This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
2+
REM Auto-generated on Fri Oct 18 2024
3+
REM This script performs a smoke test of the entire plugin by running a series of commands that collectively
44
REM hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
55
REM approximately stable.
66
REM DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
77
REM SCRIPT FROM THE PROJECT ROOT!
88
SET EXE_NAME=%1
9-
echo "At this point in the alpha, the smoke tests are no-ops and it is fine."
9+
echo "====== SETUP FOR SMOKE TESTS ======"
10+
echo "====== Delete a a pre-existing smoke-test-results directory ======"
11+
if exist smoke-test-results rmdir /s /q smoke-test-results || exit /b 1
12+
echo "====== Create smoke-test-results directory ======"
13+
if not exist smoke-test-results mkdir smoke-test-results || exit /b 1
14+
15+
16+
echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"
17+
echo "====== Show default config for all engines ======"
18+
call %EXE_NAME% code-analyzer config || exit /b 1
19+
echo "====== Show default config for PMD only ======"
20+
call %EXE_NAME% code-analyzer config -r pmd || exit /b 1
21+
echo "====== Write PMD's default config to a file ======"
22+
call %EXE_NAME% code-analyzer config -r pmd -f smoke-test-results/pmd-only-config.yml || exit /b 1
23+
echo "====== Show configuration from last step's output file ======"
24+
call %EXE_NAME% code-analyzer config -c smoke-test-results/pmd-only-config.yml || exit /b 1
25+
echo "====== Show configuration from pre-existing config file ======"
26+
call %EXE_NAME% code-analyzer config -c smoke-tests/test-data/config-files/existing-config.yml || exit /b 1
27+
28+
29+
echo "====== SMOKE TESTS FOR RULES COMMAND ======"
30+
echo "====== List all rules ======"
31+
call %EXE_NAME% code-analyzer rules || exit /b 1
32+
echo "====== List ESLint rules only ======"
33+
call %EXE_NAME% code-analyzer rules -r eslint || exit /b 1
34+
echo "====== List RetireJS rules only ======"
35+
call %EXE_NAME% code-analyzer rules -r retire-js || exit /b 1
36+
echo "====== List rules relevant to apex-only workspace ======"
37+
call %EXE_NAME% code-analyzer rules -w smoke-tests/test-data/workspace-with-apex-files || exit /b 1
38+
echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"
39+
call %EXE_NAME% code-analyzer rules -r asdfasdfasdf || exit /b 1
40+
echo "====== List rule overridden in config file ======"
41+
call %EXE_NAME% code-analyzer rules -r no-unsafe-assignment -c smoke-tests/test-data/config-files/existing-config.yml || exit /b 1
42+
43+
44+
echo "====== SMOKE TESTS FOR RUN COMMAND ======"
45+
echo "====== Run all rules against a folder ======"
46+
call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
47+
echo "====== Run all rules against a file ======"
48+
call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files/my-script.ts || exit /b 1
49+
echo "====== Run all rules against a folder and write to outfiles ======"
50+
call %EXE_NAME% code-analyzer run -w smoke-tests/test-data/workspace-with-apex-files -f smoke-test-results/outfile.json -f smoke-test-results/outfile.html || exit /b 1
51+
echo "====== Run a selection of rules against a folder ======"
52+
call %EXE_NAME% code-analyzer run -r regex -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
53+
echo "====== Run rules using a config file with overrides ======"
54+
call %EXE_NAME% code-analyzer run -c smoke-tests/test-data/config-files/existing-config.yml -w smoke-tests/test-data/workspace-with-mixed-files || exit /b 1
55+
56+
57+
echo "====== CONCLUSION ======"
58+
echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"

‎smoke-tests/smoke-test.sh‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,59 @@
11
#!/bin/bash
2-
# Auto-generated on Tue Aug 27 2024
3-
# This script WILL EVENTUALLY run a smoke test of the entire plugin by running a series of commands that collectively
2+
# Auto-generated on Fri Oct 18 2024
3+
# This script performs a smoke test of the entire plugin by running a series of commands that collectively
44
# hit a vertical slice of all major features. If they all succeed, then we can reasonably assume that the plugin is
55
# approximately stable.
66
# DO NOT EDIT THIS SCRIPT DIRECTLY! INSTEAD, MAKE CHANGES IN ./smoke-tests/smoke-test-generator.js AND RERUN THAT
77
# SCRIPT FROM THE PROJECT ROOT!
88
set -e
99
EXE_NAME=$1
10-
echo "At this point in the alpha, the smoke tests are no-ops and it is fine."
10+
echo "====== SETUP FOR SMOKE TESTS ======"
11+
echo "====== Delete a a pre-existing smoke-test-results directory ======"
12+
rm -rf smoke-test-results
13+
echo "====== Create smoke-test-results directory ======"
14+
mkdir -p smoke-test-results
15+
16+
17+
echo "====== SMOKE TESTS FOR CONFIG COMMAND ======"
18+
echo "====== Show default config for all engines ======"
19+
$EXE_NAME code-analyzer config
20+
echo "====== Show default config for PMD only ======"
21+
$EXE_NAME code-analyzer config -r pmd
22+
echo "====== Write PMD's default config to a file ======"
23+
$EXE_NAME code-analyzer config -r pmd -f smoke-test-results/pmd-only-config.yml
24+
echo "====== Show configuration from last step's output file ======"
25+
$EXE_NAME code-analyzer config -c smoke-test-results/pmd-only-config.yml
26+
echo "====== Show configuration from pre-existing config file ======"
27+
$EXE_NAME code-analyzer config -c smoke-tests/test-data/config-files/existing-config.yml
28+
29+
30+
echo "====== SMOKE TESTS FOR RULES COMMAND ======"
31+
echo "====== List all rules ======"
32+
$EXE_NAME code-analyzer rules
33+
echo "====== List ESLint rules only ======"
34+
$EXE_NAME code-analyzer rules -r eslint
35+
echo "====== List RetireJS rules only ======"
36+
$EXE_NAME code-analyzer rules -r retire-js
37+
echo "====== List rules relevant to apex-only workspace ======"
38+
$EXE_NAME code-analyzer rules -w smoke-tests/test-data/workspace-with-apex-files
39+
echo "====== List rules matching a nonsensical selector (i.e. list no rules) ======"
40+
$EXE_NAME code-analyzer rules -r asdfasdfasdf
41+
echo "====== List rule overridden in config file ======"
42+
$EXE_NAME code-analyzer rules -r no-unsafe-assignment -c smoke-tests/test-data/config-files/existing-config.yml
43+
44+
45+
echo "====== SMOKE TESTS FOR RUN COMMAND ======"
46+
echo "====== Run all rules against a folder ======"
47+
$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files
48+
echo "====== Run all rules against a file ======"
49+
$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-mixed-files/my-script.ts
50+
echo "====== Run all rules against a folder and write to outfiles ======"
51+
$EXE_NAME code-analyzer run -w smoke-tests/test-data/workspace-with-apex-files -f smoke-test-results/outfile.json -f smoke-test-results/outfile.html
52+
echo "====== Run a selection of rules against a folder ======"
53+
$EXE_NAME code-analyzer run -r regex -w smoke-tests/test-data/workspace-with-mixed-files
54+
echo "====== Run rules using a config file with overrides ======"
55+
$EXE_NAME code-analyzer run -c smoke-tests/test-data/config-files/existing-config.yml -w smoke-tests/test-data/workspace-with-mixed-files
56+
57+
58+
echo "====== CONCLUSION ======"
59+
echo "If you are seeing this message, the smoke tests ran successfully, and all is (approximately) well"

‎smoke-tests/test-data/config-files/existing-config.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
2+
engines:
3+
regex:
4+
custom_rules:
5+
NoTodos:
6+
regex: "/todo/gi"
7+
file_extensions:
8+
- ".ts"
9+
description: "detects todos"
10+
tags: ["recommended", "beep"]
11+
12+
rules:
13+
pmd:
14+
WhileLoopsMustUseBraces:
15+
severity: HIGH
16+
tags:
17+
- CodeStyle
18+
- Recommended
19+
- apexLanguage
20+
- Beep

‎smoke-tests/test-data/workspace-with-apex-files/MyClass1.cls‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
public class MyClass1 {
2+
public static void doSomeQueries(string[] accNames) {
3+
for (string accName : accNames) {
4+
Account acc = [SELECT Name FROM Account LIMIT 1];
5+
}
6+
}
7+
}

‎smoke-tests/test-data/workspace-with-apex-files/MyClass2.cls‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
public class MyClass2 {
2+
public static void doSomeDebugging() {
3+
System.debug(LoggingLevel.ERROR, 'aasdfasdfasdasdf');
4+
}
5+
}

‎smoke-tests/test-data/workspace-with-apex-files/MyClass3.cls‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
public class MyClass3 {
2+
public static void declareUnusedVariables() {
3+
Account acc = null;
4+
boolean b = false;
5+
string s = 'asdfasdfasdf';
6+
}
7+
}

‎smoke-tests/test-data/workspace-with-mixed-files/MyClass1.cls‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
public class MyClass1 {
2+
public static void doSomeQueries(string[] accNames) {
3+
for (string accName : accNames) {
4+
Account acc = [SELECT Name FROM Account LIMIT 1];
5+
}
6+
}
7+
}

‎smoke-tests/test-data/workspace-with-mixed-files/my-script.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
// TODO This is just something to trip a Regex Rule looking for this kind of statement
2+
3+
console.log('asdfasdfasdsasdf');
4+
const s: string = 'asdfasdf';
5+
6+
console.log(`s is ${s}`);

0 commit comments

Comments
 (0)