Merge pull request #376 from henrygab/TravisAllWarnings

Improve Travis Builds (inc. additional warnings)

Author: ladyada

.travis.yml

@@ -1,3 +1,4 @@
+sudo: required
 language: c
 dist: xenial
 git:
@@ -8,10 +9,28 @@ env:
   global:
     - BSP_PATH="$HOME/.arduino15/packages/adafruit/hardware/nrf52"
   jobs:
-    # Split into one job per board (aka variant)
-    - VARIANT="feather52840"
-    - VARIANT="cplaynrf52840"
-    - VARIANT="feather52832"
+    # empty env required, else allow_failures will SILENTY IGNORE matching against env...
+    -
+
+jobs:
+  fast_finish: true
+  include:
+    - name: "Feather 52840"
+      env:  VARIANT="feather52840"
+    - name: "Feather 52840 (All warnings)"
+      env:  ALL_WARNINGS="true" VARIANT="feather52840"
+    - name: "Circuit Playground 52840"
+      env:  VARIANT="cplaynrf52840"
+    - name: "Circuit Playground 52840 (All warnings)"
+      env:  ALL_WARNINGS="true" VARIANT="cplaynrf52840"
+    - name: "Feather 52832"
+      env:  VARIANT="feather52832"
+    - name: "Feather 52832 (All warnings)"
+      env:  ALL_WARNINGS="true" VARIANT="feather52832"
+  allow_failures:
+    - env:  ALL_WARNINGS="true" VARIANT="feather52840"
+    - env:  ALL_WARNINGS="true" VARIANT="cplaynrf52840"
+    - env:  ALL_WARNINGS="true" VARIANT="feather52832"
 
 addons:
   apt:
@@ -24,6 +43,10 @@ addons:
       confinement: classic
 
 install:
+  # Filter only mDNS / Bonjour traffic
+  - sudo iptables --insert INPUT  --jump DROP --protocol udp --dport 5353  -m comment --comment "silently drop all 5353/udp input"
+  - sudo iptables --insert INPUT  --jump DROP --destination 224.0.0.251    -m comment --comment "silently drop all mDNS ipv4 broadcast"
+  # Install the nRF52 support files for arduino
   - pip3 install --user adafruit-nrfutil
   - umake electronics arduino $HOME/arduino_ide
   - export PATH=$HOME/arduino_ide:$PATH

tools/build_all.py

@@ -8,6 +8,13 @@
 if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
     travis = True
 
+all_warnings = False
+if "ALL_WARNINGS" in os.environ and os.environ["ALL_WARNINGS"] == "true":
+    all_warnings = True
+
+ENV_VARIABLE_NAME = 'VARIANT'
+
+
 exit_status = 0
 success_count = 0
 fail_count = 0
@@ -21,6 +28,25 @@
     'feather52832': 'Feather nRF52832'
 }
 
+# STDERR receives output that starts with the following text, none of which should be considered a warning or error...
+output_to_ignore = (
+    'Picked up JAVA_TOOL_OPTIONS:',
+    'Loading configuration...',
+    'Initializing packages...',
+    'Preparing boards...',
+    'Verifying...',
+)
+
+def errorOutputFilter(line):
+    if len(line) == 0:
+        return False
+    if line.isspace(): # Note: empty string does not match here!
+        return False
+    if line.startswith(output_to_ignore): # alternatively, can trim() each line, but that would create lots of short-lived strings...
+        return False
+    # TODO: additional items to remove?
+    return True
+
 
 def build_examples(variant):
     global exit_status, success_count, fail_count, build_format, build_separator
@@ -33,18 +59,39 @@ def build_examples(variant):
     print(build_separator)
     subprocess.run("arduino --board adafruit:nrf52:{}:softdevice={},debug=l0 --save-prefs".format(variant, 's140v6' if variant != 'feather52832' else 's132v6'), shell=True,
                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    
+    if all_warnings:
+        subprocess.run("arduino --pref 'compiler.warning_level=all' --save-prefs", shell=True,
+                       stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     for sketch in glob.iglob('libraries/**/*.ino', recursive=True):
         start_time = time.monotonic()
 
         if os.path.exists(os.path.dirname(sketch) + '/.skip') or os.path.exists(os.path.dirname(sketch) + '/.skip.' + variant):
             success = "skipped"
         else:
-            build_result = subprocess.run("arduino --verify {}".format(sketch), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            # TODO - preferably, would have STDERR show up in **both** STDOUT and STDERR.
+            #        preferably, would use Python logging handler to get both distinct outputs and one merged output
+            #        for now, split STDERR when building with all warnings enabled, so can detect warning/error output.
+            if all_warnings:
+                build_result = subprocess.run("arduino --verify {}".format(sketch), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            else:
+                build_result = subprocess.run("arduino --verify {}".format(sketch), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+            # get stderr into a form where len(warningLines) indicates a true warning was output to stderr
+            warningLines = [];
+            if all_warnings and build_result.stderr:
+                tmpWarningLines = build_result.stderr.decode("utf-8").splitlines()
+                warningLines = list(filter(errorOutputFilter, (tmpWarningLines)))
+
             if build_result.returncode != 0:
                 exit_status = build_result.returncode
                 success = "\033[31mfailed\033[0m   "
                 fail_count += 1
+            elif len(warningLines) != 0:
+                exit_status = -1
+                success = "\033[31mwarnings\033[0m "
+                fail_count += 1
             else:
                 success = "\033[32msucceeded\033[0m"
                 success_count += 1
@@ -56,16 +103,21 @@ def build_examples(variant):
 
         print((build_format + '| {:5.2f}s |').format(sketch.split(os.path.sep)[1], os.path.basename(sketch), success, build_duration))
 
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
+        if success != "skipped":
+            if build_result.returncode != 0:
+                print(build_result.stdout.decode("utf-8"))
+                if (build_result.stderr):
+                    print(build_result.stderr.decode("utf-8"))
+            if len(warningLines) != 0:
+                for line in warningLines:
+                    print(line)
 
         if travis:
             print('travis_fold:end:build-{}\\r'.format(sketch))
 
 
 build_time = time.monotonic()
 
-ENV_VARIABLE_NAME = 'VARIANT'
 
 # build only one variant if the environment variable is specified
 if (ENV_VARIABLE_NAME in os.environ):
@@ -74,7 +126,7 @@ def build_examples(variant):
     if (variant in variants_dict):
         build_examples(variant)
     else:
-        print('\033[31failed\033[0m - invalid variant name "{}"'.format(variant))
+        print('\033[31INTERNAL ERR\033[0m - invalid variant name "{}"'.format(variant))
         fail_count += 1
         exit_status = -1