Merge pull request #1038 from sanger/Y25-298-mongo-8-support

Y25-298 - Mongo 8.0 support

Author: BenTopping

.github/workflows/automated_release.yml

@@ -41,9 +41,9 @@ jobs:
           --tag docker.pkg.github.com/${IMAGE_NAME}:${BRANCH_NAME}
 
       - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.7.0
+        uses: supercharge/mongodb-github-action@1.12.0
         with:
-          mongodb-version: 4.2
+          mongodb-version: 8.0
           mongodb-replica-set: heron_rs
 
       - name: Create SQL Server testing database

.github/workflows/ci.yml

@@ -135,20 +135,19 @@ jobs:
         run: |
           pipenv sync --dev --system
       - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.7.0
+        uses: supercharge/mongodb-github-action@1.12.0
         with:
-          mongodb-version: 4.2
+          mongodb-version: 8.0
           mongodb-replica-set: heron_rs
       - name: Setup the test MLWH and Events databases
+        run: |
+          python setup_test_db.py
+      - name: Create SQL Server testing database
         run: |
           curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
           curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           sudo apt-get update
           sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
-          sudo apt-get install -y unixodbc-dev
-          python setup_test_db.py
-      - name: Create SQL Server testing database
-        run: |
           python setup_sqlserver_test_db.py
       - name: Test with pytest
         run: |

Dockerfile

@@ -20,11 +20,15 @@ RUN apt-get update && \
     unixodbc-dev
 
 # Install the Microsoft ODBC driver for SQL Server
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
-    curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+#   https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16&tabs=debian18-install%2Calpine17-install%2Cdebian8-install%2Credhat7-13-install%2Crhel7-offline#18
+# Download the package to configure the Microsoft repo
+RUN curl -sSL -O https://packages.microsoft.com/config/debian/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2 | cut -d '.' -f 1)/packages-microsoft-prod.deb
+# Install the package
+RUN dpkg -i packages-microsoft-prod.deb
+# Delete the file
+RUN rm packages-microsoft-prod.deb
 RUN apt-get update
 RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18
-RUN apt-get install -y unixodbc-dev
 
 # Install the package manager - pipenv
 RUN pip install --upgrade pip && \

dependencies/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   # A MySQL service as close to production as currently possible
   # To run it as a standalone container:
@@ -16,9 +15,9 @@ services:
 
   # A MongoDB service as close to production as currently possible
   # To run it as a standalone container:
-  #   docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:4.2
-  mongo-4.2:
-    image: mongo:4.2
+  #   docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:8.0
+  mongo-8.0:
+    image: mongo:8.0
     container_name: mongo
     ports:
       - "27017:27017"
@@ -62,9 +61,9 @@ services:
   #   docker network create redpanda-network
   #   docker volume create redpanda-volume
   #   docker run --name=schema_registry --net=redpanda-network -v "redpanda-volume:/var/lib/redpanda/data" -p 8081:8081 \
-  #     docker.vectorized.io/vectorized/redpanda start --overprovisioned --smp 1 --memory 250M
+  #     redpandadata/redpanda start --overprovisioned --smp 1 --memory 250M
   schema_registry:
-    image: docker.vectorized.io/vectorized/redpanda
+    image: redpandadata/redpanda
     networks:
       - redpanda-network
     ports:

dependencies/scripts/mongo-init-replica-set.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-mongo <<EOF
+mongosh <<EOF
 var config = {
     _id: "heron_rs",
     members: [{ _id: 0, host: "mongo:27017"}]

dependencies/up.sh

@@ -6,4 +6,4 @@ docker compose up -d
 # Initialise the MongoDB replica set after 5 seconds
 sleep 5
 printf "\n\nInitialising the MongoDB replica set...\n\n"
-docker exec mongo /scripts/rs-init.sh
+docker exec mongo /bin/bash /scripts/rs-init.sh

tests/fixtures/data/positive_samples_in_source_plate.py

@@ -16,7 +16,7 @@
 )
 from tests.fixtures.data.source_plates import SOURCE_PLATES
 
-DATE_TESTED_NOW = datetime.utcnow()
+DATE_TESTED_NOW = datetime.now()
 
 POSITIVE_SAMPLES = [
     {

tests/fixtures/data/samples.py

@@ -17,7 +17,7 @@
 )
 from tests.fixtures.data.source_plates import SOURCE_PLATES
 
-DATE_TESTED_NOW = datetime.utcnow()
+DATE_TESTED_NOW = datetime.now()
 # NOTE: Remember that the samples of 'plate_123' are joined to the priority samples
 #   There should be 7 fit to pick samples from all the plates below
 SAMPLES = [