Move CI designs to a separate repository (#1867)

~ Move test designs to a separate repository and add it as a submodule.
~ Adjust test set to refer to designs in submodule.
~ Change design_dir in `run_tests.py` for artifact upload in the CI.
~ Fix fail condition on design name mismatch in `compare_regression_reports.py`
~ Print a better error message for design name mismatch mentioned previously. 
~ Change regression benchmark to point reflect submodule change. 
~ Change documentation to reflect new paths of designs.
+ Add aes_user_project_wrapper to ci designs to test EXTRA_SPEFS.
+ Add a step in CI workflow to replace `/` with `_` in design name for artifact upload.

Author: kareefardi

.github/scripts/run_tests.py

@@ -98,8 +98,9 @@ def cat(x):
 
 
 print("Tarballing run...")
+design_dir = design if os.path.exists(design) else os.path.join("designs", design)
 subprocess.check_call(
-    ["tar", "-czf", "./reproducible.tar.gz", os.path.join("designs", design, "runs")]
+    ["tar", "-czf", "./reproducible.tar.gz", os.path.join(design_dir, "runs")]
 )
 print("Created ./reproducible.tar.gz.")
 

.github/test_sets/test_sets.yml

@@ -1,41 +1,42 @@
 - scl: sky130A/sky130_fd_sc_hd
   name: fastest_test_set
   designs:
-    - inverter
-    - wbqspiflash
-    - APU
-    - s44
-    - zipdiv
-    - xtea
-    - usb
-    - usb_cdc_core
-    - manual_macro_placement_test
+    - ./designs/ci/inverter
+    - ./designs/ci/wbqspiflash
+    - ./designs/ci/APU
+    - ./designs/ci/s44
+    - ./designs/ci/zipdiv
+    - ./designs/ci/xtea
+    - ./designs/ci/usb
+    - ./designs/ci/usb_cdc_core
+    - ./designs/ci/manual_macro_placement_test
     - spm
-    - gcd
-    - caravel_upw
+    - ./designs/ci/gcd
+    - ./designs/ci/caravel_upw
+    - ./designs/ci/aes_user_project_wrapper
 - scl: sky130A/sky130_fd_sc_hd
   name: extended_test_set
   designs:
-    - BM64
-    - salsa20
-    - aes_core
-    - blabla
-    - picorv32a
-    - PPU
-    - y_huff
-    - aes
+    - ./designs/ci/BM64
+    - ./designs/ci/salsa20
+    - ./designs/ci/aes_core
+    - ./designs/ci/blabla
+    - ./designs/ci/picorv32a
+    - ./designs/ci/PPU
+    - ./designs/ci/y_huff
+    - ./designs/ci/aes
 - scl: gf180mcuC/gf180mcu_fd_sc_mcu7t5v0
   name: fastest_test_set
   designs:
     - spm
-    - APU
-    - usb
+    - ./designs/ci/APU
+    - ./designs/ci/usb
     # - usb_cdc_core
     # - zipdiv
     # - wbqspiflash
 - scl: gf180mcuC/gf180mcu_fd_sc_mcu7t5v0
   name: extended_test_set
   designs:
-    - picorv32a
-    - PPU
-    - y_huff
+    - ./designs/ci/picorv32a
+    - ./designs/ci/PPU
+    - ./designs/ci/y_huff

.github/workflows/openlane_ci.yml

@@ -179,6 +179,8 @@ jobs:
       matrix: ${{ fromJSON(needs.prepare_test_matrices.outputs.design_matrix) }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
 
       - name: Set up environment variables
         uses: ./.github/actions/set_env_variables
@@ -220,11 +222,17 @@ jobs:
           OPENLANE_IMAGE_NAME=$OPENLANE_IMAGE_NAME-amd64\
             python3 ${GITHUB_WORKSPACE}/.github/scripts/run_tests.py ${{ matrix.design.name }}
 
+      - name: Escape Design Name
+        run: |
+          design_name=${{ matrix.design.name }}
+          escaped_design_name=${design_name//\//_}
+          echo "ESCAPED_DESIGN_NAME=$escaped_design_name" >> $GITHUB_ENV
+
       - name: Upload Run Tarball
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.design.name }}-${{ matrix.design.pdk }}
+          name: ${{ env.ESCAPED_DESIGN_NAME }}-${{ matrix.design.pdk }}
           path: ./reproducible.tar.gz
 
   cleanup_and_deploy:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "designs/ci"]
+	path = designs/ci
+	url = https://github.com/efabless/openlane-ci-designs