FLS-1473 Build optimisation changes

.devcontainer/setup.sh

@@ -10,19 +10,10 @@ echo "--  Pulling git submodules into local  --"
 git pull --recurse-submodules
 
 echo "--  Installing digital-form-builder-adapter locally  --"
-node update-package.js
 yarn install
 
-echo "--  Building digital-form-builder-adapter locally  --"
-yarn setup
-
-echo "--  Building digital-form-builder locally  --"
-# shellcheck disable=SC2164
-cd digital-form-builder
-yarn
-
 echo "--  Building digital-form-builder model locally  --"
-yarn model build
+yarn digital-form-builder/model build yarn model build
 
 echo "--  Building digital-form-builder queue-model locally  --"
-yarn queue-model build
+yarn digital-form-builder/queue-model build

.github/workflows/copilot_deploy.yml

@@ -69,6 +69,13 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Remove submodule dockerignore / yarn.lock (if present)
+        if: ${{ hashFiles('digital-form-builder/.dockerignore') != '' || hashFiles('digital-form-builder/yarn.lock') != '' }}
+        run: |
+          rm -f digital-form-builder/.dockerignore digital-form-builder/yarn.lock
 
       - name: "Set version in env"
         id: set-version
@@ -95,68 +102,17 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: actions/checkout@v4
-      - name: Use Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "20.x"
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn config get cacheFolder)"
-
-      - name: Get into the directory
-        id: change-dir
-        run: yarn config set enableImmutableInstalls false
-
-      - name: Create .env for runner workspace
-        run: |
-          touch ./runner/.env
-          echo LAST_TAG_GH=runner >> ./runner/.env
-          echo LAST_COMMIT_GH=runner >> ./runner/.env
-          cat ./runner/.env
-
-      - name: Run Unit test before docker build
-        run: |
-          echo "--  Git submodule initialize into local  --"
-          git submodule update --init
-          echo "--  Pulling git submodules into local  --"
-          git pull --recurse-submodules
-          echo "--  Installing digital-form-builder-adapter locally  --"
-          node update-package.js
-          yarn install
-          echo "--  Building digital-form-builder-adapter locally  --"
-          yarn setup
-          echo "--  Building digital-form-builder locally  --"
-          # shellcheck disable=SC2164
-          cd digital-form-builder
-          yarn
-          echo "--  Building digital-form-builder model locally  --"
-          yarn model build
-          echo "--  Building digital-form-builder queue-model locally  --"
-          yarn queue-model build
-          echo "--  Building digital-form-builder-adapter model locally  --"
-          cd ..
-          yarn model build
-          echo "--  Running unit tests  --"
-          yarn runner test-cov
-
       - name: Set up Docker Buildx property
         run: echo "DOCKER_BUILDKIT=1" >> $GITHUB_ENV
 
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: ~/.buildx/cache
-          key: ${{ runner.os }}-buildx-runner-${{ hashFiles('./runner/Dockerfile') }}-${{ hashFiles('./runner/package.json') }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-runner-${{ hashFiles('./runner/Dockerfile') }}-${{ hashFiles('./runner/package.json') }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
           driver: docker-container
 
+      - name: Cache stats (Buildx)
+        run: docker buildx du --verbose || true
+
       - name: Build and push docker image
         uses: docker/build-push-action@v4
         with:
@@ -168,16 +124,22 @@ jobs:
           build-args: |
             INSTALL_NODEMON=false
             LAST_TAG='${{env.VERSION}}'
-            LAST_COMMIT='${{ github.sha }}'
-          cache-from: type=gha,scope=buildx
-          cache-to: type=gha,mode=max,scope=buildx
+          cache-from: type=gha,scope=runner
+          cache-to: type=gha,mode=max,scope=runner
 
   docker-designer-build:
     runs-on: ubuntu-latest
     needs: [ setup ]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Remove submodule dockerignore / yarn.lock (if present)
+        if: ${{ hashFiles('digital-form-builder/.dockerignore') != '' || hashFiles('digital-form-builder/yarn.lock') != '' }}
+        run: |
+          rm -f digital-form-builder/.dockerignore digital-form-builder/yarn.lock
 
       - name: "Set version in env"
         id: set-version
@@ -211,19 +173,14 @@ jobs:
           echo LAST_COMMIT_GH=designer >> ./designer/.env
           cat ./designer/.env
 
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        with:
-          path: ~/.buildx/cache
-          key: ${{ runner.os }}-buildx-runner-${{ hashFiles('./designer/Dockerfile') }}-${{ hashFiles('./designer/package.json') }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-designer-${{ hashFiles('./designer/Dockerfile') }}-${{ hashFiles('./designer/package.json') }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:
           driver: docker-container
 
+      - name: Cache stats (Buildx)
+        run: docker buildx du --verbose || true
+
       - name: Build and push docker image
         uses: docker/build-push-action@v4
         with:
@@ -234,9 +191,8 @@ jobs:
           file: ./designer/Dockerfile
           build-args: |
             LAST_TAG='${{env.VERSION}}'
-            LAST_COMMIT='${{ github.sha }}'
-          cache-from: type=gha,scope=buildx
-          cache-to: type=gha,mode=max,scope=buildx
+          cache-from: type=gha,scope=designer
+          cache-to: type=gha,mode=max,scope=designer
 
   e2e-test:
     needs: [ setup, docker-runner-build, docker-designer-build ]
@@ -252,14 +208,21 @@ jobs:
       hash: ${{ steps.hashFile.outputs.hash }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
       - name: Use Node.js
         uses: actions/setup-node@v3
         with:
           node-version: "20.x"
 
-      - name: Get into the directory
-        id: change-dir
-        run: yarn config set enableImmutableInstalls false
+      - name: Use Yarn 3.2.2 via Corepack
+        run: |
+          corepack enable
+          corepack prepare [email protected] --activate
+          yarn -v
+
+      - name: Ensure workspace-tools plugin
+        run: yarn plugin import workspace-tools
 
       - uses: actions/cache@v3
         id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
@@ -274,18 +237,10 @@ jobs:
 
       - name: Setup e2e dependencies
         run: |
-          echo "--  Git submodule initialize into local  --"
-          git submodule update --init
-          echo "--  Pulling git submodules into local  --"
-          git pull --recurse-submodules
-          echo "--  Installing digital-form-builder-adapter locally  --"
-          node update-package.js
+          echo "--  Installing dependencies  --"
           yarn install
           echo "--  Install e2e dependencies  --"
           yarn e2e-test install
-          echo "--  Building digital-form-builder locally  --"
-          cd digital-form-builder
-          yarn
 
       - name: Injecting image into docker compose e2e
         run: |
@@ -469,7 +424,7 @@ jobs:
       app_name: forms
       environment: test
       notify_slack: true
-  
+
   uat_deploy:
     needs: [ setup, docker-designer-build, docker-runner-build, dev_deploy, test_deploy, post_test_deploy_tests ]
     if: ${{ always() && contains(fromJSON(needs.setup.outputs.jobs_to_run), 'uat') && (! contains(needs.*.result, 'failure') ) && (! contains(needs.*.result, 'cancelled') ) }}

.yarnrc.yml

@@ -1,5 +1,5 @@
 nodeLinker: node-modules
-
+cacheFolder: .yarn/cache
 plugins:
   - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
     spec: "@yarnpkg/plugin-workspace-tools"

README.md

@@ -10,7 +10,7 @@
 2. Make sure you have yarn 1.22+ installed. You do not need to install yarn 2.4+, yarn will detect the yarn 2 binary within [.yarn](./.yarn) and that will be used.
 3. If using the designer:
    - Note that the designer requires the runner to be running with the default `NODE_ENV=development` settings (see [runner/config/development.json](https://github.com/XGovFormBuilder/digital-form-builder/tree/main/runner/config/development.json)) to enable posting and previewing of forms during design.
-4. Run `sh .devcontainer/setup.sh` command to install all dependencies in all workspaces and create the working environment.
+4. Run `yarn setup` command to install all dependencies in all workspaces and create the working environment.
 
 
 As already mentioned, **always run scripts from the root directory.** because workspaces don't have scripts or packages you need to run from inside their folders and by running in the root directory yarn 2 can resolve the scripts/packages properly.

designer/.dockerignore

@@ -0,0 +1,12 @@
+.git
+.gitmodules
+**/node_modules
+**/.next
+**/dist
+**/build
+**/coverage
+**/.DS_Store
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.pnp.*