fix: re-add tokenserver/spanner scripts to the final docker (#2083)

poetry export doesn't install them, so these still need to be copied
over. deployments expect the tokenserver tools in
/app/tools/tokenserver

the spanner scripts are not in active use yet but adding them back
anyway

and install cargo nextest via binaries (as building it requires recent
stable rustcs)

Closes STOR-489

Author: pjenvey

.circleci/config.yml

@@ -128,8 +128,11 @@ commands:
   install-test-deps:
     steps:
       - run:
-          name: Install test dependencies
-          command: cargo install --locked cargo-nextest cargo-llvm-cov
+          name: Install cargo-nextest
+          command: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+      - run:
+          name: Install cargo-llvm-cov
+          command: cargo install --locked cargo-llvm-cov
 
   make-test-dir:
     steps:

.github/workflows/mysql.yml

@@ -69,8 +69,11 @@ jobs:
             "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             > version.json
 
-      - name: Install test dependencies
-        run: cargo install --locked cargo-nextest cargo-llvm-cov
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+
+      - name: Install cargo-llvm-cov
+        run: cargo install --locked cargo-llvm-cov
 
       - name: Run unit tests with coverage
         run: make test_with_coverage

.github/workflows/postgres.yml

@@ -67,8 +67,11 @@ jobs:
             "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             > version.json
 
-      - name: Install test dependencies
-        run: cargo install --locked cargo-nextest cargo-llvm-cov
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+
+      - name: Install cargo-llvm-cov
+        run: cargo install --locked cargo-llvm-cov
 
       - name: Run unit tests with coverage
         run: make postgres_test_with_coverage

.github/workflows/spanner.yml

@@ -67,8 +67,11 @@ jobs:
             "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
             > version.json
 
-      - name: Install test dependencies
-        run: cargo install --locked cargo-nextest cargo-llvm-cov
+      - name: Install cargo-nextest
+        run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+
+      - name: Install cargo-llvm-cov
+        run: cargo install --locked cargo-llvm-cov
 
       - name: Build workspace (spanner feature)
         run: |

Dockerfile

@@ -81,7 +81,7 @@ RUN poetry export --no-interaction --without dev --output requirements.txt --wit
     fi && \
     cd /app
 
-# Build wheels for all Python dependencies so 
+# Build wheels for all Python dependencies so
 RUN mkdir -p /app/wheels && \
     pip3 wheel --no-cache-dir -r /app/requirements.txt -w /app/wheels && \
     pip3 wheel --no-cache-dir -r /app/tools/integration_tests/requirements.txt -w /app/wheels && \
@@ -154,7 +154,9 @@ RUN pip3 install --break-system-packages --no-cache-dir --no-index --find-links=
 
 COPY --from=builder /app/bin /app/bin
 COPY --from=builder /app/version.json /app
+COPY --from=builder /app/tools/spanner /app/tools/spanner
 COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
+COPY --from=builder /app/tools/tokenserver /app/tools/tokenserver
 COPY --from=builder --chmod=0755 /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
 COPY --from=builder --chmod=0755 /app/scripts/start_mock_fxa_server.sh /app/scripts/start_mock_fxa_server.sh
 COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl