Merge pull request #34 from DataKitchen/release/3.7.6

Release/3.7.6

Author: rboni-dk

deploy/install_linuxodbc.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env sh
+
+# From: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+# modifications: Added --non-interactive and --no-cache flags, removed sudo, added aarch64 as an alias for arm64
+
+architecture="unsupported"
+
+case "$(uname -m)" in
+  x86_64)
+    architecture="amd64"
+    ;;
+  arm64|aarch64)
+    architecture="arm64"
+    ;;
+esac
+
+if [ "unsupported" = "$architecture" ]; then
+  echo "Alpine architecture $(uname -m) is not currently supported."
+  exit 1
+fi
+
+#Download the desired package(s)
+curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_$architecture.apk
+curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_$architecture.apk
+
+#(Optional) Verify signature, if 'gpg' is missing install it using 'apk add gnupg':
+curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_$architecture.sig
+curl -O https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_$architecture.sig
+
+curl https://packages.microsoft.com/keys/microsoft.asc  | gpg --import -
+gpg --verify msodbcsql18_18.4.1.1-1_$architecture.sig msodbcsql18_18.4.1.1-1_$architecture.apk
+gpg --verify mssql-tools18_18.4.1.1-1_$architecture.sig mssql-tools18_18.4.1.1-1_$architecture.apk
+
+#Install the package(s)
+apk add --no-cache --non-interactive --allow-untrusted msodbcsql18_18.4.1.1-1_$architecture.apk
+apk add --no-cache --non-interactive --allow-untrusted mssql-tools18_18.4.1.1-1_$architecture.apk

deploy/testgen-base.dockerfile

@@ -15,6 +15,9 @@ RUN apk update && apk upgrade && apk add --no-cache \
     musl-dev \
     gfortran \
     linux-headers=6.6-r0 \
+    # Tools needed for installing the MSSQL ODBC drivers \
+    curl \
+    gpg \
     # Additional libraries needed and their dev counterparts. We add both so that we can remove
     # the *-dev later, keeping the libraries
     openblas=0.3.28-r0 \
@@ -28,6 +31,9 @@ RUN apk add --no-cache \
     libarrow=18.1.0-r0 \
     apache-arrow-dev=18.1.0-r0
 
+COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh
+RUN /tmp/dk/install_linuxodbc.sh
+
 # Install TestGen's main project empty pyproject.toml to install (and cache) the dependencies first
 COPY ./pyproject.toml /tmp/dk/pyproject.toml
 RUN mkdir /dk
@@ -40,7 +46,11 @@ RUN apk del \
     cmake \
     musl-dev \
     gfortran \
+    curl \
+    gpg \
     linux-headers \
     openblas-dev \
     unixodbc-dev \
     apache-arrow-dev
+
+RUN rm /tmp/dk/install_linuxodbc.sh

deploy/testgen.dockerfile

@@ -1,11 +1,14 @@
-ARG TESTGEN_BASE_LABEL=v3
+ARG TESTGEN_BASE_LABEL=v4
 
 FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 
 # Args have to be set in current build stage: https://github.com/moby/moby/issues/37345
 ARG TESTGEN_VERSION
 ARG TESTGEN_DOCKER_HUB_REPO
 
+ENV PYTHONPATH=/dk/lib/python3.12/site-packages
+ENV PATH=$PATH:/dk/bin
+
 # Now install everything
 COPY . /tmp/dk/
 RUN python3 -m pip install --prefix=/dk /tmp/dk
@@ -17,12 +20,9 @@ RUN addgroup -S testgen && adduser -S testgen -G testgen
 RUN mkdir /var/lib/testgen
 RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/streamlit/static
 
-ENV PYTHONPATH=/dk/lib/python3.12/site-packages
-ENV PATH=$PATH:/dk/bin
-
 ENV TESTGEN_VERSION=${TESTGEN_VERSION}
-ENV TG_RELEASE_CHECK=docker
 ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO}
+ENV TG_RELEASE_CHECK=docker
 ENV STREAMLIT_SERVER_MAX_UPLOAD_SIZE=200
 
 USER testgen

invocations/dev.py

@@ -14,8 +14,6 @@
 
 DOCKER_BUILDER_NAME = "dk-builder"
 DOCKER_BUILDER_PLATFORMS = "linux/amd64,linux/arm64"
-TESTGEN_DEFAULT_BASE_LABEL = "v3"
-
 
 @task
 def required_tools(ctx: Context) -> None:
@@ -91,7 +89,7 @@ def build_public_image(
     version: str = "",
 
     load: bool = False,
-    base_label: str = TESTGEN_DEFAULT_BASE_LABEL,
+    base_label: str = "",
     debug: bool = False
 ) -> None:
     """Builds and pushes the TestGen image"""
@@ -114,12 +112,12 @@ def build_public_image(
     if debug:
         extra_args.append("--print")
 
-    ctx.run(
-        f"docker buildx bake -f deploy/docker-bake.hcl testgen-{target} {' '.join(extra_args)} ",
-        env={
-            "TESTGEN_LABELS": " ".join(label),
-            "TESTGEN_BASE_LABEL": base_label,
-            "TESTGEN_VERSION": version,
-        },
-        echo=True,
-    )
+    env={
+        "TESTGEN_LABELS": " ".join(label),
+        "TESTGEN_VERSION": version,
+    }
+    if base_label:
+        env["TESTGEN_BASE_LABEL"] = base_label
+
+    cmd = f"docker buildx bake -f deploy/docker-bake.hcl testgen-{target} {' '.join(extra_args)} "
+    ctx.run(cmd, env=env, echo=True)

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "3.7.1"
+version = "3.7.6"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "[email protected]" },

testgen/ui/services/test_definition_service.py

@@ -134,6 +134,7 @@ def validate_test(test_definition):
         connection["connect_by_key"],
         connection["private_key"],
         connection["private_key_passphrase"],
+        connection["http_path"],
         sql_query,
     )