Add support for adding release info into adafruit/circuitpython-org

This also changes the build script to python with better output.

Author: tannewt

.gitmodules

@@ -83,3 +83,6 @@
 [submodule "tools/huffman"]
 	path = tools/huffman
 	url = https://github.com/tannewt/huffman.git
+[submodule "tools/adabot"]
+	path = tools/adabot
+	url = https://github.com/adafruit/adabot.git

.travis.yml

@@ -21,18 +21,29 @@ git:
 # that SDK is shortest and add it there.  In the case of major re-organizations,
 # just try to make the builds "about equal in run time"
 env:
-  - TRAVIS_TESTS="unix docs translations" TRAVIS_BOARDS="feather_huzzah circuitplayground_express pca10056 pca10059 feather_nrf52832 feather_nrf52840_express makerdiary_nrf52840_mdk" TRAVIS_SDK=arm:nrf:esp8266
+  - TRAVIS_TESTS="unix docs translations website" TRAVIS_BOARDS="feather_huzzah circuitplayground_express pca10056 pca10059 feather_nrf52832 feather_nrf52840_express makerdiary_nrf52840_mdk" TRAVIS_SDK=arm:nrf:esp8266
   - TRAVIS_BOARDS="metro_m0_express metro_m4_express pirkey_m0 trellis_m4_express trinket_m0" TRAVIS_SDK=arm
   - TRAVIS_BOARDS="feather_radiofruit_zigbee gemma_m0 hallowing_m0_express itsybitsy_m0_express itsybitsy_m4_express meowmeow" TRAVIS_SDK=arm
   - TRAVIS_BOARDS="feather_m0_express_crickit feather_m0_rfm69 feather_m0_rfm9x feather_m4_express arduino_zero arduino_mkr1300" TRAVIS_SDK=arm
-  - TRAVIS_BOARDS="circuitplayground_express_crickit feather_m0_adalogger feather_m0_basic feather_m0_express catwan_usbstick" TRAVIS_SDK=arm 
+  - TRAVIS_BOARDS="circuitplayground_express_crickit feather_m0_adalogger feather_m0_basic feather_m0_express catwan_usbstick" TRAVIS_SDK=arm
 
 addons:
   artifacts:
     paths:
     - $(ls -d1 bin/*/* | tr "\n" ":")
     target_paths: /
 
+deploy:
+  provider: releases
+  api_key:
+    secure: "jdqVFw6itRY4qwQF4ReXo0uaymT+Mob6RhYX0lw8KWFNqBgHnLVuKmKKcGMEuRvBVMPkvxF7bMuOQzSBOunqwlHFse3oMzdWvQODv1zwV7pSRXGwTdIvTPbBjKWxnBG9uSNRf2R5AMflJFUxy2CbtBpgvNzr+4VLREZDrrjEu8C1iTtXGpSm5AQ5iIp2fkMAWD85FP7CQPpkqRoxhSIFZmTdurfYRmenq1OZ/4SeD5TESKcyvqJNzVT9z210B3cg3eAkP6ukvelW4qE2zgIANqUkGqvDEnAvEII9M89kuwhCMAekdfwnUSPrry+g77i1dUZHoRN1+MFj+waYtPaqxdYo2G1sysa6enxlu4jHMR5MfMk9eKHgaNgL3PiyANusYSS44amh8QIiVaX5nw82myZDCpQOZW7YqJKE6WX70Lbs4mS+wIs+ig4KIXO1B0p9kMb0OeVjHRl+KcXsWGRu/ECG/ExpqlVIssSPU407LohMXT2cJ37CY/R/EeK2XSDsQ2M3L3EAGUjCJdBGuwsOJ+2lG+HQpAVu9vAB4kq5jy9Ye+MG+8Xlkly3XZZ5+FkXyYxKnXb26/QVv0e5sIG5OmdJCPYFaH2J1QdKo7CdhEcBtrf6DMPWaimGMldShFqzLjOz3b3qLysRxFF0aGb7ipKPa57vawNzYHoPAViOcXQ="
+  file:
+  - $(ls -d1 bin/*/* | tr "\n" ":")
+  skip_cleanup: true
+  draft: true
+  on:
+    tags: true
+
 notifications:
   webhooks:
     urls:
@@ -46,12 +57,13 @@ notifications:
 before_script:
   # Expand the git tree back to 4.0.0-alpha.1 and then fetch the latest tag.
   - LAST_TAG=`git ls-remote --quiet --tags --sort=version:refname | egrep -o "refs/tags/[0-9]+.*\$" | tail -n 1`
-  - git fetch --shallow-exclude=4.0.0-alpha.1
+  - git fetch --shallow-exclude=4.0.0-alpha.1 || git fetch --unshallow
   - git fetch --depth 1 origin $LAST_TAG:$LAST_TAG
   - git describe --dirty --always --tags
   - function var_search () { case "$1" in *$2*) true;; *) false;; esac; }
   - sudo dpkg --add-architecture i386
 
+
   - (! var_search "${TRAVIS_SDK-}" arm || (wget https://s3.amazonaws.com/adafruit-circuit-python/gcc-arm-embedded_7-2018q2-1~xenial1_amd64.deb && sudo dpkg -i gcc-arm-embedded*_amd64.deb))
 
   # For nrf builds
@@ -62,10 +74,11 @@ before_script:
   - if var_search "${TRAVIS_SDK-}" esp8266 ; then PATH=$(readlink -f ../xtensa-lx106-elf/bin):$PATH; fi
 
   # For coverage testing (upgrade is used to get latest urllib3 version)
-  - ([[ -z "$TRAVIS_TESTS" ]] || sudo apt-get install -y python3-pip)
+  - sudo apt-get install -y python3-pip
+  - pip3 install --user sh click
   - ([[ -z "$TRAVIS_TESTS" ]] || sudo pip install --upgrade cpp-coveralls)
-  - (! var_search "${TRAVIS_TESTS-}" docs || sudo pip install 'Sphinx<1.8.0' sphinx-rtd-theme recommonmark)
-  - (! var_search "${TRAVIS_TESTS-}" translations || sudo pip3 install polib)
+  - (! var_search "${TRAVIS_TESTS-}" docs || pip install --user 'Sphinx<1.8.0' sphinx-rtd-theme recommonmark)
+  - (! var_search "${TRAVIS_TESTS-}" translations || pip3 install --user polib)
 
   # report some good version numbers to the build
   - gcc --version
@@ -79,9 +92,8 @@ script:
   - make -C mpy-cross -j2
   - echo -en 'travis_fold:end:mpy-cross\\r'
 
-  - echo 'Building Adafruit binaries' && echo -en 'travis_fold:start:adafruit-bins\\r'
-  - (for board in $TRAVIS_BOARDS; do TRAVIS_BOARD=$board tools/build_adafruit_bins.sh || exit $?; done)
-  - echo -en 'travis_fold:end:adafruit-bins\\r'
+  - cd tools && python3 build_adafruit_bins.py
+  - cd ..
 
   - echo 'Building unix' && echo -en 'travis_fold:start:unix\\r'
   - (! var_search "${TRAVIS_TESTS-}" unix || (make -C ports/unix deplibs -j2 && make -C ports/unix -j2 && make -C ports/unix coverage -j2))
@@ -119,5 +131,7 @@ script:
   # run coveralls coverage analysis (try to, even if some builds/tests failed)
   #- (cd ports/unix && coveralls --root ../.. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
 
+  - (! var_search "${TRAVIS_TESTS-}" website || (cd tools && python3 build_board_info.py && cd ..))
+
 after_failure:
   - (cd tests && for exp in *.exp; do testbase=$(basename $exp .exp); echo -e "\nFAILURE $testbase"; diff -u $testbase.exp $testbase.out; done)

ports/esp8266/.gitignore

@@ -0,0 +1 @@
+build-*/

ports/esp8266/Makefile

@@ -6,7 +6,7 @@ $(error Invalid BOARD specified)
 endif
 
 # If the build directory is not given, make it reflect the board name.
-BUILD ?= build
+BUILD ?= build-$(BOARD)
 
 include ../../py/mkenv.mk
 
@@ -25,7 +25,7 @@ FROZEN_MPY_DIR ?= modules
 # include py core make definitions
 include $(TOP)/py/py.mk
 
-FWBIN = $(BUILD)/firmware-combined.bin
+FWBIN = $(BUILD)/firmware.bin
 PORT ?= /dev/ttyACM0
 BAUD ?= 115200
 FLASH_MODE ?= qio
@@ -232,7 +232,7 @@ FROZEN_EXTRA_DEPS = $(CONFVARS_FILE)
 
 .PHONY: deploy
 
-deploy: $(BUILD)/firmware-combined.bin
+deploy: $(FWBIN)
 	$(ECHO) "Writing $< to the board"
 	$(Q)$(ESPTOOL) --port $(PORT) --baud $(BAUD) write_flash --verify --flash_size=$(FLASH_SIZE) --flash_mode=$(FLASH_MODE) 0 $<
 
@@ -275,4 +275,4 @@ $(BUILD)/libaxtls.a:
 
 clean-modules:
 	git clean -f -d modules
-	rm -f build/frozen*.c
+	rm -f $(BUILD)/frozen*.c

ports/esp8266/esp8266_common.ld

@@ -142,7 +142,7 @@ SECTIONS
         *lib/utils/*.o*(.literal*, .text*)
         *drivers/bus/*.o(.literal* .text*)
 
-        build/main.o(.literal* .text*)
+        build*/main.o(.literal* .text*)
         *gccollect.o(.literal* .text*)
         *gchelper.o(.literal* .text*)
         *help.o(.literal* .text*)
@@ -178,7 +178,7 @@ SECTIONS
         */frozen.o(.rodata.mp_frozen_content) /* frozen modules */
 
         /* for -mforce-l32 */
-        build/*.o(.rodata*)
+        build*/*.o(.rodata*)
 
         _irom0_text_end = ABSOLUTE(.);
     } >irom0_0_seg :irom0_0_phdr

ports/nrf/Makefile

@@ -10,12 +10,14 @@ else
   endif
 endif
 
+CLI_SD := $(SD)
+
 include boards/$(BOARD)/mpconfigboard.mk
 
 SD_LOWER = $(shell echo $(SD) | tr '[:upper:]' '[:lower:]')
 
-# Build directory with SD
-BUILD ?= $(if $(SD),build-$(BOARD)-$(SD_LOWER),build-$(BOARD))
+# Build directory with SD if it's different from the default.
+BUILD ?= $(if $(CLI_SD),build-$(BOARD)-$(SD_LOWER),build-$(BOARD))
 
 include ../../py/mkenv.mk
 -include mpconfigport.mk

ports/unix/Makefile

@@ -74,7 +74,7 @@ else
 # Use gcc syntax for map file
 LDFLAGS_ARCH = -Wl,-Map=$@.map,--cref -Wl,--gc-sections
 endif
-LDFLAGS = $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
+LDFLAGS = -Lbuild $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
 
 ifeq ($(MICROPY_FORCE_32BIT),1)
 # Note: you may need to install i386 versions of dependency packages,
@@ -245,7 +245,7 @@ coverage:
 	$(MAKE) \
 	    COPT="-O0" CFLAGS_EXTRA='$(CFLAGS_EXTRA) -DMP_CONFIGFILE="<mpconfigport_coverage.h>" \
 	    -fprofile-arcs -ftest-coverage \
-	    -Wdouble-promotion -Wformat -Wmissing-declarations -Wmissing-prototypes -Wsign-compare \
+	    -Wformat -Wmissing-declarations -Wmissing-prototypes -Wsign-compare \
 	    -Wold-style-definition -Wpointer-arith -Wshadow -Wuninitialized -Wunused-parameter \
 	    -DMICROPY_UNIX_COVERAGE' \
 	    LDFLAGS_EXTRA='-fprofile-arcs -ftest-coverage' \

py/py.mk

@@ -28,7 +28,7 @@ ifeq ($(MICROPY_PY_USSL),1)
 CFLAGS_MOD += -DMICROPY_PY_USSL=1
 ifeq ($(MICROPY_SSL_AXTLS),1)
 CFLAGS_MOD += -DMICROPY_SSL_AXTLS=1 -I$(TOP)/lib/axtls/ssl -I$(TOP)/lib/axtls/crypto -I$(TOP)/lib/axtls/config
-LDFLAGS_MOD += -Lbuild -laxtls
+LDFLAGS_MOD += -L$(BUILD) -laxtls
 else ifeq ($(MICROPY_SSL_MBEDTLS),1)
 # Can be overridden by ports which have "builtin" mbedTLS
 MICROPY_SSL_MBEDTLS_INCLUDE ?= $(TOP)/lib/mbedtls/include

tools/adabot

@@ -0,0 +1,70 @@
+import os
+from sh import rm
+from sh import make
+import sys
+import subprocess
+import shutil
+import build_board_info as build_info
+import time
+
+for port in build_info.SUPPORTED_PORTS:
+    rm("-rf", "../ports/{}/build*".format(port))
+
+ROSIE_SETUPS = ["rosie-ci"]
+rosie_ok = {}
+for rosie in ROSIE_SETUPS:
+    rosie_ok[rosie] = True
+
+PARALLEL = "-j 5"
+travis = False
+if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
+    PARALLEL="-j 2"
+    travis = True
+
+all_boards = build_info.get_board_mapping()
+build_boards = list(all_boards.keys())
+if "TRAVIS_BOARDS" in os.environ:
+    build_boards = os.environ["TRAVIS_BOARDS"].split()
+
+sha, version = build_info.get_version_info()
+
+languages = build_info.get_languages()
+exit_status = 0
+for board in build_boards:
+    bin_directory = "../bin/{}/".format(board)
+    os.makedirs(bin_directory, exist_ok=True)
+    board_info = all_boards[board]
+
+    for language in languages:
+        start_time = time.monotonic()
+        make_result = subprocess.run("make -C ../ports/" + board_info["port"] + " TRANSLATION=" + language + " BOARD=" + board, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        build_duration = time.monotonic() - start_time
+        success = "\033[32msucceeded\033[0m"
+        if make_result.returncode != 0:
+            exit_status = make_result.returncode
+            success = "\033[31mfailed\033[0m"
+
+        for extension in board_info["extensions"]:
+            temp_filename = "../ports/{port}/build-{board}/firmware.{extension}".format(port=board_info["port"], board=board, extension=extension)
+            final_filename = "adafruit-circuitpython-{board}-{language}-{version}.{extension}".format(board=board, language=language, version=version, extension=extension)
+            final_filename = os.path.join(bin_directory, final_filename)
+            shutil.copyfile(temp_filename, final_filename)
+
+        if travis:
+            print('travis_fold:start:adafruit-bins-{}-{}\\r'.format(language, board))
+        print("Build {} for {} took {:.2f}s and {}".format(board, language, build_duration, success))
+        print(len(make_result.stdout))
+        print(make_result.stdout.decode("utf-8"))
+        # Only upload to Rosie if its a pull request.
+        if travis:
+            for rosie in ROSIE_SETUPS:
+                if not rosie_ok[rosie]:
+                    break
+                print("Uploading to https://{rosie}.ngrok.io/upload/{sha}".format(rosie=rosie, sha=sha))
+                #curl -F "file=@$final_filename" https://$rosie.ngrok.io/upload/$sha
+        if travis:
+            print('travis_fold:end:adafruit-bins-{}-{}\\r'.format(language, board))
+
+        print()
+
+sys.exit(exit_status)