If #[itest(focus)] is encountered, fail memcheck CI jobs

Bromeon · Bromeon · commit 1a4a47a9ee89 · 2023-03-06T21:01:38.000+01:00
Prevents accidental disabling of tests in CI (during development).
diff --git a/.github/composite/godot-install/action.yml b/.github/composite/godot-install/action.yml
@@ -10,7 +10,7 @@ inputs:
     required: true
     description: "Name of the compiled Godot artifact to download"
 
-  binary-filename:
+  godot-binary:
     required: true
     description: "Filename of the Godot executable"
 
@@ -25,7 +25,7 @@ runs:
       run: |
         runnerDir=$(echo "${{ runner.temp }}" | sed "s!\\\\!/!")
         echo "RUNNER_DIR=$runnerDir" >> $GITHUB_ENV
-        echo "GODOT4_BIN=$runnerDir/godot_bin/${{ inputs.binary-filename }}" >> $GITHUB_ENV
+        echo "GODOT4_BIN=$runnerDir/godot_bin/${{ inputs.godot-binary }}" >> $GITHUB_ENV
       shell: bash
 
 #    - name: "Check cache for installed Godot version"
diff --git a/.github/composite/godot-itest/action.yml b/.github/composite/godot-itest/action.yml
@@ -10,10 +10,15 @@ inputs:
     required: true
     description: "Name of the compiled Godot artifact to download"
 
-  binary-filename:
+  godot-binary:
     required: true
     description: "Filename of the Godot executable"
 
+  godot-args:
+    required: false
+    default: ''
+    description: "Command-line arguments passed to Godot"
+
   rust-toolchain:
     required: false
     default: 'stable'
@@ -26,8 +31,8 @@ inputs:
 
   with-llvm:
     required: false
-    description: "Set to 'true' if LLVM should be installed"
     default: ''
+    description: "Set to 'true' if LLVM should be installed"
 
 
 runs:
@@ -39,7 +44,7 @@ runs:
       uses: ./.github/composite/godot-install
       with:
         artifact-name: ${{ inputs.artifact-name }}
-        binary-filename: ${{ inputs.binary-filename }}
+        godot-binary: ${{ inputs.godot-binary }}
 
     # The chmod seems still necessary, although applied before uploading artifact. Possibly modes are not preserved.
     # The `| xargs` pattern trims the output, since printed version may contain extra newline, which causes problems in env vars.
@@ -94,7 +99,7 @@ runs:
       run: |
         cd itest/godot
         echo "OUTCOME=itest" >> $GITHUB_ENV
-        $GODOT4_BIN --headless 2>&1 | tee "${{ runner.temp }}/log.txt" | tee >(grep "SCRIPT ERROR:" -q && {
+        $GODOT4_BIN --headless ${{ inputs.godot-args }} 2>&1 | tee "${{ runner.temp }}/log.txt" | tee >(grep "SCRIPT ERROR:" -q && {
         	printf "\n -- Godot engine encountered error, abort...\n";
         	pkill godot
             echo "OUTCOME=godot-runtime" >> $GITHUB_ENV
diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml
@@ -57,7 +57,7 @@ jobs:
         uses: ./.github/composite/godot-install
         with:
           artifact-name: godot-linux
-          binary-filename: godot.linuxbsd.editor.dev.x86_64
+          godot-binary: godot.linuxbsd.editor.dev.x86_64
 
       - name: "Check clippy"
         run: |
@@ -126,7 +126,7 @@ jobs:
         uses: ./.github/composite/godot-install
         with:
           artifact-name: godot-${{ matrix.name }}
-          binary-filename: ${{ matrix.godot-binary }}
+          godot-binary: ${{ matrix.godot-binary }}
 
       - name: "Compile tests"
         run: cargo test $GDEXT_FEATURES --no-run
@@ -168,15 +168,19 @@ jobs:
           # Additionally, the Godot source is patched to make dlclose() a no-op, as unloading dynamic libraries loses stacktrace and
           # cause false positives like println!. See https://github.com/google/sanitizers/issues/89.
           # The gcc version can possibly be removed later, as it is slower and needs a larger artifact than the clang one.
+
+          # --disallow-focus: fail if #[itest(focus)] is encountered, to prevent running only a few tests for full CI
           - name: linux-memcheck-gcc
             os: ubuntu-20.04
             rust-toolchain: stable
             godot-binary: godot.linuxbsd.editor.dev.x86_64.san
+            godot-args: -- --disallow-focus
 
           - name: linux-memcheck-clang
             os: ubuntu-20.04
             rust-toolchain: stable
             godot-binary: godot.linuxbsd.editor.dev.x86_64.llvm.san
+            godot-args: -- --disallow-focus
 
     steps:
       - uses: actions/checkout@v3
@@ -185,7 +189,8 @@ jobs:
         uses: ./.github/composite/godot-itest
         with:
           artifact-name: godot-${{ matrix.name }}
-          binary-filename: ${{ matrix.godot-binary }}
+          godot-binary: ${{ matrix.godot-binary }}
+          godot-args: ${{ matrix.godot-args }}
           with-llvm: ${{ matrix.name == 'macos' }}
 
 
diff --git a/.github/workflows/minimal-ci.yml b/.github/workflows/minimal-ci.yml
@@ -57,7 +57,7 @@ jobs:
         uses: ./.github/composite/godot-install
         with:
           artifact-name: godot-linux
-          binary-filename: godot.linuxbsd.editor.dev.x86_64
+          godot-binary: godot.linuxbsd.editor.dev.x86_64
 
       - name: "Check clippy"
         run: |
@@ -78,7 +78,7 @@ jobs:
         uses: ./.github/composite/godot-install
         with:
           artifact-name: godot-linux
-          binary-filename: godot.linuxbsd.editor.dev.x86_64
+          godot-binary: godot.linuxbsd.editor.dev.x86_64
 
       - name: "Compile tests"
         run: cargo test $GDEXT_FEATURES --no-run
@@ -98,7 +98,7 @@ jobs:
         uses: ./.github/composite/godot-itest
         with:
           artifact-name: godot-linux
-          binary-filename: godot.linuxbsd.editor.dev.x86_64
+          godot-binary: godot.linuxbsd.editor.dev.x86_64
           #godot_ver: ${{ matrix.godot }}
 
 
diff --git a/itest/godot/TestRunner.gd b/itest/godot/TestRunner.gd
@@ -5,6 +5,20 @@
 extends Node
 
 func _ready():
+	var allow_focus := true
+	var unrecognized_args: Array = []
+	for arg in OS.get_cmdline_user_args():
+		match arg:
+			"--disallow-focus":
+				allow_focus = false
+			_:
+				unrecognized_args.push_back(arg)
+
+	if unrecognized_args:
+		push_error("Unrecognized arguments: ", unrecognized_args)
+		get_tree().quit(2)
+		return
+
 	var rust_runner = IntegrationTests.new()
 
 	var gdscript_suites: Array = [
@@ -19,7 +33,7 @@ func _ready():
 			if method_name.begins_with("test_"):
 				gdscript_tests.push_back(GDScriptTestCase.new(suite, method_name))
 
-	var success: bool = rust_runner.run_all_tests(gdscript_tests, gdscript_suites.size())
+	var success: bool = rust_runner.run_all_tests(gdscript_tests, gdscript_suites.size(), allow_focus)
 
 	var exit_code: int = 0 if success else 1
 	get_tree().quit(exit_code)
diff --git a/itest/rust/src/runner.rs b/itest/rust/src/runner.rs
@@ -23,16 +23,18 @@ pub(crate) struct IntegrationTests {
 impl IntegrationTests {
     #[allow(clippy::uninlined_format_args)]
     #[func]
-    fn run_all_tests(&mut self, gdscript_tests: Array, gdscript_file_count: i64) -> bool {
-        println!(
-            "{}Run{} Godot integration tests...",
-            FMT_GREEN_BOLD, FMT_END
-        );
+    fn run_all_tests(
+        &mut self,
+        gdscript_tests: Array,
+        gdscript_file_count: i64,
+        allow_focus: bool,
+    ) -> bool {
+        println!("{}Run{} Godot integration tests...", FMT_CYAN_BOLD, FMT_END);
 
         let (rust_tests, rust_file_count, focus_run) = super::collect_rust_tests();
         self.focus_run = focus_run;
         if focus_run {
-            println!("  Focused run -- execute only selected Rust tests.")
+            println!("  {FMT_CYAN}Focused run{FMT_END} -- execute only selected Rust tests.")
         }
         println!(
             "  Rust: found {} tests in {} files.",
@@ -57,7 +59,7 @@ impl IntegrationTests {
             None
         };
 
-        self.conclude(rust_time, gdscript_time)
+        self.conclude(rust_time, gdscript_time, allow_focus)
     }
 
     fn run_rust_tests(&mut self, tests: Vec<RustTestCase>) {
@@ -87,7 +89,12 @@ impl IntegrationTests {
         }
     }
 
-    fn conclude(&self, rust_time: Duration, gdscript_time: Option<Duration>) -> bool {
+    fn conclude(
+        &self,
+        rust_time: Duration,
+        gdscript_time: Option<Duration>,
+        allow_focus: bool,
+    ) -> bool {
         let Self {
             total,
             passed,
@@ -103,10 +110,11 @@ impl IntegrationTests {
 
         let rust_time = rust_time.as_secs_f32();
         let gdscript_time = gdscript_time.map(|t| t.as_secs_f32());
+        let focused_run = gdscript_time.is_none();
 
         let extra = if skipped > 0 {
             format!(", {skipped} skipped")
-        } else if gdscript_time.is_none() {
+        } else if focused_run {
             " (focused run)".to_string()
         } else {
             "".to_string()
@@ -121,7 +129,13 @@ impl IntegrationTests {
         } else {
             println!("  Time: {rust_time:.2}s.");
         }
-        all_passed
+
+        if focused_run && !allow_focus {
+            println!("  {FMT_YELLOW}Focus run disallowed; return failure.{FMT_END}");
+            false
+        } else {
+            all_passed
+        }
     }
 
     fn update_stats(&mut self, outcome: &TestOutcome) {
@@ -139,7 +153,8 @@ impl IntegrationTests {
 //     use rand::seq::SliceRandom;
 //     let outcome = [TestOutcome::Passed, TestOutcome::Failed, TestOutcome::Skipped];
 //     let outcome = outcome.choose(&mut rand::thread_rng()).unwrap();
-const FMT_GREEN_BOLD: &str = "\x1b[32;1;1m";
+const FMT_CYAN_BOLD: &str = "\x1b[36;1;1m";
+const FMT_CYAN: &str = "\x1b[36m";
 const FMT_GREEN: &str = "\x1b[32m";
 const FMT_YELLOW: &str = "\x1b[33m";
 const FMT_RED: &str = "\x1b[31m";
