Introduce todo.txt annotation to mark integration tests as disabled.

Similar to previous `disabled.txt`, but ensures that fixed integration tests don't stay disabled.

Failing tests marked as "to do" will be reported as *skipped*; passing tests marked as "to do" will be reported as *failed*.

Set environment variable `METAFIX_DISABLE_TO_DO=true` (or system property `org.metafacture.metafix.disableToDo=true` when run via Gradle) to mimic previous `disabled.txt` behaviour.

Analogous to `@MetafixToDo` annotation in unit tests (see #158 and f4a7523).

Author: blackwinter

build.gradle

@@ -6,14 +6,16 @@ plugins {
 
 editorconfig {
   excludes = [
-    '**/generated',
-    '**/xtext-gen',
-    '**/bin',
+    '**/*.out',
+    '**/*.vsix',
     '**/.*',
-    'gradlew*',
-    '**/out',
+    '**/bin',
+    '**/generated',
     '**/node_modules',
-    '**/*.vsix'
+    '**/out',
+    '**/output-*',
+    '**/xtext-gen',
+    'gradlew*'
   ]
 }
 

metafix/build.gradle

@@ -63,7 +63,9 @@ test {
 
 task integrationTest(type: Exec, group: 'Verification') {
   executable './integrationTest.sh'
-  if (project.hasProperty("args")) args project.getProperty("args").split()
+
+  if (project.hasProperty('args')) args project.getProperty('args').split()
+  environment.METAFIX_DISABLE_TO_DO = System.getProperty('org.metafacture.metafix.disableToDo')
 }
 
 task install(dependsOn: publishToMavenLocal, group: 'Publishing',

metafix/integrationTest.sh

@@ -8,7 +8,7 @@ metafix_file=test.flux
 catmandu_file=test.cmd
 
 fix_file=test.fix
-disabled_file=disabled.txt
+todo_file=todo.txt
 
 input_glob=input.*
 expected_glob=expected.*
@@ -20,6 +20,12 @@ root_directory="$PWD"
 data_directory="$root_directory/src/test/resources/org/metafacture/metafix/integration"
 gradle_command="$root_directory/../gradlew"
 
+function parse_boolean() {
+  [ "${1,,}" == true ]
+}
+
+parse_boolean "$METAFIX_DISABLE_TO_DO" && disable_todo=1 || disable_todo=
+
 [ -t 1 -a -x /usr/bin/colordiff ] && colordiff=colordiff || colordiff=cat
 
 function _tput() {
@@ -112,13 +118,30 @@ function command_info() {
 
   [ -s "$3" ] && log "  ${color_info}${1^} command output$color_reset: $3" || rm -f "$3"
   [ -s "$4" ] && log "  ${color_info}${1^} command error$color_reset:  $4" || rm -f "$4"
+
+  log
+}
+
+function skip_test() {
+  if [ -r "$2" ]; then
+    local message="$color_test$1$color_reset: ${color_skipped}SKIPPED$color_reset" reason=$(head -1 "$2")
+
+    [ -n "$reason" ] && message+=" ($reason)"
+    log "$message"
+
+    ((skipped++)) || true
+
+    return 0;
+  else
+    return 1;
+  fi
 }
 
 function run_tests() {
   local test matched=1\
-    test_input test_expected test_disabled\
-    metafix_command_output metafix_command_error\
-    metafix_exit_status metafix_output metafix_diff
+    test_directory test_fix test_input test_expected test_todo\
+    metafix_command_output metafix_command_error metafix_start_time\
+    metafix_exit_status metafix_output metafix_diff metafix_elapsed_time
 
   cd "$data_directory"
 
@@ -138,13 +161,9 @@ function run_tests() {
     get_file "$test" expected "$test_directory"/$expected_glob || { log; continue; }
     test_expected=$current_file
 
-    test_disabled="$test_directory/$disabled_file"
+    test_todo="$test_directory/$todo_file"
 
-    if [ -r "$test_disabled" ]; then
-      log "$color_test$test$color_reset: ${color_skipped}SKIPPED$color_reset ($(<"$test_disabled"))"
-
-      ((skipped++)) || true
-    else
+    if [ -z "$disable_todo" ] || ! skip_test "$test" "$test_todo"; then
       metafix_command_output="$test_directory/metafix.out"
       metafix_command_error="$test_directory/metafix.err"
 
@@ -162,14 +181,18 @@ function run_tests() {
           metafix_elapsed_time=$(elapsed_time "$metafix_start_time")
 
           if diff -u "$test_expected" "$metafix_output" >"$metafix_diff"; then
-            #log "$color_test$test$color_reset: ${color_passed}PASSED$color_reset$metafix_elapsed_time"
+            if [ -r "$test_todo" ]; then
+              log "$color_test$test$color_reset: ${color_failed}FAILED$color_reset (Marked as \"to do\", but passed.)"
 
-            rm -f "$metafix_diff" "$metafix_command_output" "$metafix_command_error"
+              ((failed++)) || true
+            else
+              #log "$color_test$test$color_reset: ${color_passed}PASSED$color_reset$metafix_elapsed_time"
 
-            ((passed++)) || true
+              ((passed++)) || true
+            fi
 
-            #log
-          else
+            rm -f "$metafix_diff" "$metafix_command_output" "$metafix_command_error"
+          elif ! skip_test "$test" "$test_todo"; then
             log "$color_test$test$color_reset: ${color_failed}FAILED$color_reset$metafix_elapsed_time"
 
             log "  Fix:      $test_fix"
@@ -183,24 +206,18 @@ function run_tests() {
             command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
 
             ((failed++)) || true
-
-            log
           fi
         else
           command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
-
-          log
         fi
-      else
+      elif ! skip_test "$test" "$test_todo"; then
         metafix_exit_status=$?
 
         log "$color_test$test$color_reset: ${color_error}ERROR$color_reset"
 
         command_info metafix "$metafix_exit_status" "$metafix_command_output" "$metafix_command_error"
 
         ((failed++)) || true
-
-        log
       fi
     fi
   done