Add test files for adapters

Author: Benedikt Artelt

spec/lib/junit_adapter_spec.rb

@@ -5,24 +5,262 @@
 RSpec.describe JunitAdapter do
   let(:adapter) { described_class.new }
 
+
   describe '#parse_output' do
-    context 'with failed tests' do
+    
+    context 'CorrectSolution' do
+      #Initialize data for Testcase 2.1
+      let(:count) { 42 }
+      let(:stdout) { "OK (#{count} tests)" }
+      
+      #Testcase 2.1
+      it 'returns the correct numbers' do
+        expect(adapter.parse_output(stdout:)).to eq(count:, passed: count)
+      end
+    end
+
+    context 'with assertionerrors' do
+      #Initialize data for Testcase 2.2
       let(:count) { 42 }
       let(:failed) { 25 }
       let(:stdout) { "FAILURES!!!\nTests run: #{count},  Failures: #{failed}" }
+      let(:stderr) {""}
+      let(:error_matches) { [] }
+
 
+      #Testcase 2.2
       it 'returns the correct numbers' do
-        expect(adapter.parse_output(stdout:)).to eq(count:, failed:)
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
       end
     end
+    
+    #From here on compilation errors only (count = failed = 0)
+    context 'unclosed string literal' do
 
-    context 'without failed tests' do
-      let(:count) { 42 }
-      let(:stdout) { "OK (#{count} tests)" }
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
 
-      it 'returns the correct numbers' do
-        expect(adapter.parse_output(stdout:)).to eq(count:, passed: count)
+      #Initialize data for Testcase 2.3
+      let(:stderr) { File.read("spec/testcasedatajava/UnclosedStringLiteralTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 6** occured **unclosed string literal** at String s=\"test; "]}
+      #["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 6** occured **unclosed string literal** at String s=\"test; "]
+
+      #Testcase 2.3
+      it 'unclosed string literal' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'illegal start of expression' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.4
+      let(:stderr) { File.read("spec/testcasedatajava/IllegalStartOfExpressionTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 10** occured **illegal start of expression** at private int i=0; "]}
+
+      #Testcase 2.4
+      it 'illegal start of expression' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'variable might not have been initialized' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.5
+      let(:stderr) { File.read("spec/testcasedatajava/VariableMightNotHaveBeenInitializedTest.txt")}
+      let(:error_matches) { ["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 11** occured **variable i might not have been initialized** at i=i+1; "]}
+
+      #Testcase 2.5
+      it 'variable .. might not have been initialized' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'not a statement' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.6
+      let(:stderr) { File.read("spec/testcasedatajava/NotAStatementTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **not a statement** at \"test\"; "]}
+
+      #Testcase 2.6
+      it 'not a statement' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'unexpected type' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.7
+      let(:stderr) { File.read("spec/testcasedatajava/UnexpectedTypeTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 11** occured **unexpected type** at if(1=0){ "]}
+
+      #Testcase 2.7
+      it 'unexpected type' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'something expected' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.8
+      let(:stderr) { File.read("spec/testcasedatajava/SomethingExpectedTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **';' expected** at return 1/power(base,-exponent) "]}
+
+      #Testcase 2.8
+      it '... expected' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'bad operand types' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.9
+      let(:stderr) { File.read("spec/testcasedatajava/BadOperandTypesTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **bad operand types for binary operator '+'** at System.out.println(a+b); "]}
+
+      #Testcase 2.9
+      it 'bad operand types ...' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'incompatible types' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.10
+      let(:stderr) { File.read("spec/testcasedatajava/IncompatibleTypesTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **incompatible types: String cannot be converted to int** at power(a,s); "]}
+
+      #Testcase 2.10
+      it 'incompatible types: ...' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'missing return' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.11
+      let(:stderr) { File.read("spec/testcasedatajava/MissingReturnTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 19** occured **missing return statement** at } "]}
+
+      #Testcase 2.11
+      it 'missing return ..' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'invalid method declaration' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.12
+      let(:stderr) { File.read("spec/testcasedatajava/InvalidMethodDeclarationTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 19** occured **invalid method declaration; return type required** at public static add(int a,int b){ "]}
+ 
+      #Testcase 2.12
+      it 'invalid method declaration ...' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'unreachable statement' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.13
+      let(:stderr) { File.read("spec/testcasedatajava/UnreachableStatementTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **unreachable statement** at System.out.println(\"test\"); "]}
+ 
+      #Testcase 2.13
+      it 'unreachable statement' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'cannot be referenced from a static context' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.14
+      let(:stderr) { File.read("spec/testcasedatajava/CannotBeReferencedFromAStaticContextTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 6** occured **non-static variable field cannot be referenced from a static context** at field = parm; "]}
+ 
+      #Testcase 2.14
+      it '... cannot be referenced from a static context' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'reached end of file while parsing' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.15
+      let(:stderr) { File.read("spec/testcasedatajava/ReachedEndOfFileWhileParsingTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 22** occured **reached end of file while parsing** at } "]}
+ 
+      #Testcase 2.15
+      it 'reached end of file while parsing' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'cannot be applied to' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.16
+      let(:stderr) { File.read("spec/testcasedatajava/CannotBeAppliedToTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 12** occured **method power in class RecursiveMath cannot be applied to given types;** at return 1/power();\n required: int,int\n found: no arguments "]}
+
+      #Testcase 2.16
+      it '... cannot be applied to ...' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
+      end
+    end
+
+    context 'cannot find symbol' do
+      let(:count) {0}
+      let(:failed) {0}
+      let(:stdout) {""}
+
+      #Initialize data for Testcase 2.17
+      let(:stderr) { File.read("spec/testcasedatajava/CannotFindSymbolTest.txt")}
+      let(:error_matches) {["<span style=\"color:red\">**Compilation Error**</span> in **file** ./org/example/RecursiveMath **line 11** occured **cannot find symbol** at System.out.println(i);\n symbol:   variable i\n location: class RecursiveMath "]}
+
+      #Testcase 2.17
+      it 'cannot find symbol...' do
+        expect(adapter.parse_output(stdout: stdout, stderr: stderr)).to eq(count:, failed:, error_messages: error_matches)
       end
     end
   end
-end
+
+end

spec/lib/py_unit_adapter_spec.rb

@@ -4,13 +4,51 @@
 
 RSpec.describe PyUnitAdapter do
   let(:adapter) { described_class.new }
-  let(:count) { 42 }
-  let(:failed) { 25 }
-  let(:stderr) { "Ran #{count} tests in 0.1s\n\nFAILED (failures=#{failed})" }
 
   describe '#parse_output' do
-    it 'returns the correct numbers' do
-      expect(adapter.parse_output(stderr:)).to eq(count:, failed:)
+
+    #Initialize data for Testcase 1.1
+    let(:count) {2}
+    let(:failed) {0}
+    let(:error_matches) {[]}
+    let(:stderr) { File.read("spec/testcasedata/CorrectSolution.txt")}
+
+    #Testcase 1.1
+    it 'CorrectSolution' do
+      expect(adapter.parse_output(stderr:)).to eq(count:, failed:, error_messages: error_matches)
+    end
+
+    #Initialize data for Testcase 1.2
+    let(:count) {2}
+    let(:failed) {1}
+    let(:error_matches) {["test_odd: 0 is not true"]}
+    let(:stderr) { File.read("spec/testcasedata/correctnumber.txt")}
+
+    #Testcase 1.2
+    it 'correct count/failed number' do
+      expect(adapter.parse_output(stderr:)).to eq(count:, failed:, error_messages: error_matches)
+    end
+
+    #count and failed always 0 for following test cases (tests can't be executed)
+    let(:count) {0}
+    let(:failed) {0}
+    #Initialize data for Testcase 1.3
+    let(:error_matches) {["<span style=\"color:red\">**SyntaxError**</span>: invalid syntax in **file** /workspace/exercise.py **line 1**"]}
+    let(:stderr) { File.read("spec/testcasedata/SyntaxErrorTest.txt")}
+
+    #Testcase 1.3
+    it 'SyntaxError' do
+      expect(adapter.parse_output(stderr:)).to eq(count:, failed:, error_messages: error_matches)
+    end
+
+    #Initialize data for Testcase 1.4
+    let(:error_matches) {["<span style=\"color:red\">**IndentationError**</span>: expected an indented block in **file** /workspace/exercise.py **line 2**"]}
+    let(:stderr) { File.read("spec/testcasedata/IndentationErrorTest.txt")}
+
+    #Testcase 1.4
+    it 'IndentationError' do
+      expect(adapter.parse_output(stderr:)).to eq(count:, failed:, error_messages: error_matches)
     end
+    #TabError not tested at the moment because current codeocean version never raises it
   end
 end

spec/testcasedata/CorrectSolution.txt

@@ -0,0 +1,7 @@
+test_even (exercise_tests.ExerciseTests) ... ok
+test_odd (exercise_tests.ExerciseTests) ... ok
+
+----------------------------------------------------------------------
+Ran 2 tests in 0.000s
+
+OK

spec/testcasedata/CorrectSolutionErrors.txt

@@ -0,0 +1 @@
+[]

spec/testcasedata/IndentationErrorTest.txt

@@ -0,0 +1,25 @@
+Traceback (most recent call last):
+  File "/usr/lib/python3.4/runpy.py", line 170, in _run_module_as_main
+    "__main__", mod_spec)
+  File "/usr/lib/python3.4/runpy.py", line 85, in _run_code
+    exec(code, run_globals)
+  File "/usr/lib/python3.4/unittest/__main__.py", line 18, in <module>
+    main(module=None)
+  File "/usr/lib/python3.4/unittest/main.py", line 92, in __init__
+    self.parseArgs(argv)
+  File "/usr/lib/python3.4/unittest/main.py", line 139, in parseArgs
+    self.createTests()
+  File "/usr/lib/python3.4/unittest/main.py", line 146, in createTests
+    self.module)
+  File "/usr/lib/python3.4/unittest/loader.py", line 157, in loadTestsFromNames
+    suites = [self.loadTestsFromName(name, module) for name in names]
+  File "/usr/lib/python3.4/unittest/loader.py", line 157, in <listcomp>
+    suites = [self.loadTestsFromName(name, module) for name in names]
+  File "/usr/lib/python3.4/unittest/loader.py", line 116, in loadTestsFromName
+    module = __import__('.'.join(parts_copy))
+  File "/workspace/exercise_tests.py", line 1, in <module>
+    from exercise import *
+  File "/workspace/exercise.py", line 2
+    if(x%2)==0:
+     ^
+IndentationError: expected an indented block

spec/testcasedata/IndentationErrorTestErrors.txt

@@ -0,0 +1 @@
+["<span style=\"color:red\">**IndentationError**</span>: expected an indented block in **file** /workspace/exercise.py **line 2**"]

spec/testcasedata/SyntaxErrorTest.txt

@@ -0,0 +1,25 @@
+Traceback (most recent call last):
+  File "/usr/lib/python3.4/runpy.py", line 170, in _run_module_as_main
+    "__main__", mod_spec)
+  File "/usr/lib/python3.4/runpy.py", line 85, in _run_code
+    exec(code, run_globals)
+  File "/usr/lib/python3.4/unittest/__main__.py", line 18, in <module>
+    main(module=None)
+  File "/usr/lib/python3.4/unittest/main.py", line 92, in __init__
+    self.parseArgs(argv)
+  File "/usr/lib/python3.4/unittest/main.py", line 139, in parseArgs
+    self.createTests()
+  File "/usr/lib/python3.4/unittest/main.py", line 146, in createTests
+    self.module)
+  File "/usr/lib/python3.4/unittest/loader.py", line 157, in loadTestsFromNames
+    suites = [self.loadTestsFromName(name, module) for name in names]
+  File "/usr/lib/python3.4/unittest/loader.py", line 157, in <listcomp>
+    suites = [self.loadTestsFromName(name, module) for name in names]
+  File "/usr/lib/python3.4/unittest/loader.py", line 116, in loadTestsFromName
+    module = __import__('.'.join(parts_copy))
+  File "/workspace/exercise_tests.py", line 1, in <module>
+    from exercise import *
+  File "/workspace/exercise.py", line 1
+    ef even(x):
+          ^
+SyntaxError: invalid syntax

spec/testcasedata/SyntaxErrorTestErrors.txt

@@ -0,0 +1 @@
+["<span style=\"color:red\">**SyntaxError**</span>: invalid syntax in **file** /workspace/exercise.py **line 1**"]