Merge pull request #1039 from sanger/develop

Develop -> Master

Author: BenTopping

.github/workflows/automated_release.yml

@@ -41,9 +41,9 @@ jobs:
           --tag docker.pkg.github.com/${IMAGE_NAME}:${BRANCH_NAME}
 
       - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.7.0
+        uses: supercharge/mongodb-github-action@1.12.0
         with:
-          mongodb-version: 4.2
+          mongodb-version: 8.0
           mongodb-replica-set: heron_rs
 
       - name: Create SQL Server testing database

.github/workflows/check_release_version.yml

@@ -7,21 +7,5 @@ on:
       - master
 
 jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v41
-        with:
-          files: |
-            .release-version
-          
-      - name: Run step looking for change in the release version
-        run: >-
-          if ! "${{ steps.changed-files-specific.outputs.any_changed }}"; then
-            echo "Please change the release version number"
-            exit 1;
-          fi
+  check-release-version:
+    uses: sanger/.github/.github/workflows/check-release-version.yml@master

.github/workflows/ci.yml

@@ -135,20 +135,19 @@ jobs:
         run: |
           pipenv sync --dev --system
       - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.7.0
+        uses: supercharge/mongodb-github-action@1.12.0
         with:
-          mongodb-version: 4.2
+          mongodb-version: 8.0
           mongodb-replica-set: heron_rs
       - name: Setup the test MLWH and Events databases
+        run: |
+          python setup_test_db.py
+      - name: Create SQL Server testing database
         run: |
           curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
           curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
           sudo apt-get update
           sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
-          sudo apt-get install -y unixodbc-dev
-          python setup_test_db.py
-      - name: Create SQL Server testing database
-        run: |
           python setup_sqlserver_test_db.py
       - name: Test with pytest
         run: |

.release-version

@@ -1 +1 @@
-2.37.0
+2.38.0

Dockerfile

@@ -20,11 +20,15 @@ RUN apt-get update && \
     unixodbc-dev
 
 # Install the Microsoft ODBC driver for SQL Server
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
-    curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+#   https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16&tabs=debian18-install%2Calpine17-install%2Cdebian8-install%2Credhat7-13-install%2Crhel7-offline#18
+# Download the package to configure the Microsoft repo
+RUN curl -sSL -O https://packages.microsoft.com/config/debian/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2 | cut -d '.' -f 1)/packages-microsoft-prod.deb
+# Install the package
+RUN dpkg -i packages-microsoft-prod.deb
+# Delete the file
+RUN rm packages-microsoft-prod.deb
 RUN apt-get update
 RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18
-RUN apt-get install -y unixodbc-dev
 
 # Install the package manager - pipenv
 RUN pip install --upgrade pip && \

Pipfile

@@ -19,18 +19,18 @@ types-requests = "*"
 colorlog = "~=6.8"
 eve = "~=2.1"
 flask-apscheduler = "~=1.13"
-flask-cors = "~=4.0"
-gunicorn = "~=22.0"
+flask-cors = "~=5.0"
+gunicorn = "~=23.0"
 openpyxl = "~=3.1"
 pandas = "~=2.0"
 pika = "~=1.3"
 pymysql = "~=1.1"
 pyodbc = "~=5.1"
-python-dotenv = "~=1.0"
+python-dotenv = "~=1.1"
 requests = "~=2.32"
 slackclient = "~=2.9"
 sqlalchemy = "~=2.0"
-pymongo = "~=4.8.0"
+pymongo = "~=4.12.1"
 setuptools = "*"
 numpy = "*"
 

Pipfile.lock

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   # A MySQL service as close to production as currently possible
   # To run it as a standalone container:
@@ -16,9 +15,9 @@ services:
 
   # A MongoDB service as close to production as currently possible
   # To run it as a standalone container:
-  #   docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:4.2
-  mongo-4.2:
-    image: mongo:4.2
+  #   docker run --name mongo -d -p 27017:27017 --entrypoint="/usr/bin/mongod --bind_ip_all --replSet heron_rs" mongo:8.0
+  mongo-8.0:
+    image: mongo:8.0
     container_name: mongo
     ports:
       - "27017:27017"
@@ -62,9 +61,9 @@ services:
   #   docker network create redpanda-network
   #   docker volume create redpanda-volume
   #   docker run --name=schema_registry --net=redpanda-network -v "redpanda-volume:/var/lib/redpanda/data" -p 8081:8081 \
-  #     docker.vectorized.io/vectorized/redpanda start --overprovisioned --smp 1 --memory 250M
+  #     redpandadata/redpanda start --overprovisioned --smp 1 --memory 250M
   schema_registry:
-    image: docker.vectorized.io/vectorized/redpanda
+    image: redpandadata/redpanda
     networks:
       - redpanda-network
     ports:

dependencies/docker-compose.yml

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-mongo <<EOF
+mongosh <<EOF
 var config = {
     _id: "heron_rs",
     members: [{ _id: 0, host: "mongo:27017"}]

dependencies/scripts/mongo-init-replica-set.sh

@@ -6,4 +6,4 @@ docker compose up -d
 # Initialise the MongoDB replica set after 5 seconds
 sleep 5
 printf "\n\nInitialising the MongoDB replica set...\n\n"
-docker exec mongo /scripts/rs-init.sh
+docker exec mongo /bin/bash /scripts/rs-init.sh