Merge pull request #31 from DataKitchen/release/3.7.1

Release/3.7.1

Author: datakitchen-devops

deploy/install_arrow.sh

@@ -6,77 +6,41 @@ ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONFAULTHANDLER=1
 ENV ACCEPT_EULA=Y
 
-RUN apk update && apk add --no-cache \
+RUN apk update && apk upgrade && apk add --no-cache \
+    # Tools needed for building the python wheels
     gcc \
     g++ \
-    bash \
-    libffi-dev \
-    openssl-dev \
-    cargo \
-    musl-dev \
-    postgresql-dev \
+    make \
     cmake \
-    rust \
-    linux-headers \
-    libc-dev \
-    libgcc \
-    libstdc++ \
-    ca-certificates \
-    zlib-dev \
-    bzip2-dev \
-    xz-dev \
-    lz4-dev \
-    zstd-dev \
-    snappy-dev \
-    brotli-dev \
-    build-base \
-    autoconf \
-    boost-dev \
-    flex \
-    libxml2-dev \
-    libxslt-dev \
-    libjpeg-turbo-dev \
-    ninja \
-    git \
-    curl \
-    unixodbc-dev \
-    gpg \
-    openssl=3.3.2-r1 \
+    musl-dev \
     gfortran \
-    openblas-dev
-
-RUN mkdir /dk
+    linux-headers=6.6-r0 \
+    # Additional libraries needed and their dev counterparts. We add both so that we can remove
+    # the *-dev later, keeping the libraries
+    openblas=0.3.28-r0 \
+    openblas-dev=0.3.28-r0 \
+    unixodbc=2.3.12-r0 \
+    unixodbc-dev=2.3.12-r0
 
-COPY --chmod=775 ./deploy/install_linuxodbc.sh /tmp/dk/install_linuxodbc.sh
-RUN /tmp/dk/install_linuxodbc.sh
-
-COPY --chmod=775 ./deploy/install_arrow.sh /tmp/dk/install_arrow.sh
-RUN /tmp/dk/install_arrow.sh
+RUN apk add --no-cache \
+    --repository https://dl-cdn.alpinelinux.org/alpine/v3.21/community \
+    --repository https://dl-cdn.alpinelinux.org/alpine/v3.21/main \
+    libarrow=18.1.0-r0 \
+    apache-arrow-dev=18.1.0-r0
 
 # Install TestGen's main project empty pyproject.toml to install (and cache) the dependencies first
 COPY ./pyproject.toml /tmp/dk/pyproject.toml
+RUN mkdir /dk
 RUN python3 -m pip install --prefix=/dk /tmp/dk
 
 RUN apk del \
     gcc \
     g++ \
-    bash \
-    libffi-dev \
-    openssl-dev \
-    cargo \
-    musl-dev \
-    postgresql-dev \
+    make \
     cmake \
-    rust \
+    musl-dev \
+    gfortran \
     linux-headers \
-    libc-dev \
-    build-base \
-    autoconf \
-    boost-dev \
-    flex \
-    ninja \
-    curl \
+    openblas-dev \
     unixodbc-dev \
-    gpg \
-    ca-certificates \
-    git
+    apache-arrow-dev

deploy/install_linuxodbc.sh

@@ -1,38 +1,30 @@
-ARG TESTGEN_BASE_LABEL=v1
+ARG TESTGEN_BASE_LABEL=v3
 
-FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS build-image
-
-# Now install everything
-COPY . /tmp/dk/
-RUN python3 -m pip install --prefix=/dk /tmp/dk
-
-FROM python:3.12.7-alpine3.20 AS release-image
+FROM datakitchen/dataops-testgen-base:${TESTGEN_BASE_LABEL} AS release-image
 
 # Args have to be set in current build stage: https://github.com/moby/moby/issues/37345
 ARG TESTGEN_VERSION
 ARG TESTGEN_DOCKER_HUB_REPO
 
-RUN addgroup -S testgen && adduser -S testgen -G testgen
+# Now install everything
+COPY . /tmp/dk/
+RUN python3 -m pip install --prefix=/dk /tmp/dk
+RUN rm -Rf /tmp/dk
 
-COPY --from=build-image --chown=testgen:testgen /dk/ /dk
-COPY --from=build-image /usr/local/lib/ /usr/local/lib
-COPY --from=build-image /usr/lib/ /usr/lib
-COPY --from=build-image /opt/microsoft/ /opt/microsoft
-COPY --from=build-image /etc/odbcinst.ini /etc/odbcinst.ini
+RUN addgroup -S testgen && adduser -S testgen -G testgen
 
-# The OpenSSL upgrade is not carried from the build image, so we have to upgrade it again
-#RUN apk add --no-cache openssl=3.3.2-r1
+# Streamlit has to be able to write to these dirs
+RUN mkdir /var/lib/testgen
+RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/streamlit/static
 
 ENV PYTHONPATH=/dk/lib/python3.12/site-packages
-ENV PATH="$PATH:/dk/bin:/opt/mssql-tools/bin/"
+ENV PATH=$PATH:/dk/bin
 
 ENV TESTGEN_VERSION=${TESTGEN_VERSION}
 ENV TG_RELEASE_CHECK=docker
 ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO}
 ENV STREAMLIT_SERVER_MAX_UPLOAD_SIZE=200
 
-RUN mkdir /var/lib/testgen && chown testgen:testgen /var/lib/testgen
-
 USER testgen
 
 WORKDIR /dk

deploy/testgen-base.dockerfile

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "3.1.2"
+version = "3.7.1"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "[email protected]" },
@@ -31,6 +31,7 @@ dependencies = [
     "PyYAML==6.0.1",
     "click==8.1.3",
     "sqlalchemy==1.4.46",
+    "databricks-sql-connector==2.9.3",
     "snowflake-sqlalchemy==1.6.1",
     "pyodbc==5.0.0",
     "psycopg2-binary==2.9.9",
@@ -56,17 +57,19 @@ dependencies = [
     "xlsxwriter==3.2.0",
     "psutil==5.9.8",
     "concurrent_log_handler==0.9.25",
-    "cryptography==43.0.1",
+    "cryptography==44.0.1",
     "validators==0.33.0",
     "reportlab==4.2.2",
-    "pydantic==1.10.11",
+    "pydantic==1.10.13",
     "streamlit-pydantic==0.6.0",
 
     # Pinned to match the manually compiled libs or for security
-    "pyarrow==18.0.0",
-    "snowflake-connector-python==3.12.3",
+    "pyarrow==18.1.0",
+    "snowflake-connector-python==3.13.1",
     "matplotlib==3.9.2",
     "scipy==1.14.1",
+    "tornado==6.4.2",
+    "jinja2==3.1.6",
 ]
 
 [project.optional-dependencies]

deploy/testgen.dockerfile

@@ -39,9 +39,9 @@ def __init__(self, strProjectCode, strTestSuiteId, strTestSuite, strSQLFlavor, m
     def _get_rollup_scores_sql(self) -> CRollupScoresSQL:
         if not self._rollup_scores_sql:
             self._rollup_scores_sql = CRollupScoresSQL(self.test_run_id, self.table_groups_id)
-    
+
         return self._rollup_scores_sql
-    
+
     def _ReplaceParms(self, strInputString):
         strInputString = strInputString.replace("{MAX_QUERY_CHARS}", str(self.max_query_chars))
         strInputString = strInputString.replace("{TEST_RUN_ID}", self.test_run_id)
@@ -51,6 +51,7 @@ def _ReplaceParms(self, strInputString):
         strInputString = strInputString.replace("{TABLE_GROUPS_ID}", self.table_groups_id)
 
         strInputString = strInputString.replace("{SQL_FLAVOR}", self.flavor)
+        strInputString = strInputString.replace("{ID_SEPARATOR}", "`" if self.flavor == "databricks" else '"')
         strInputString = strInputString.replace("{CONCAT_OPERATOR}", self.concat_operator)
 
         strInputString = strInputString.replace("{SCHEMA_NAME}", self.target_schema)
@@ -71,8 +72,9 @@ def _ReplaceParms(self, strInputString):
 
         strInputString = replace_templated_functions(strInputString, self.flavor)
 
-        # Adding escape character where ':' is referenced
-        strInputString = strInputString.replace(":", "\\:")
+        if self.flavor != "databricks":
+            # Adding escape character where ':' is referenced
+            strInputString = strInputString.replace(":", "\\:")
 
         return strInputString
 
@@ -110,12 +112,12 @@ def PushTestRunStatusUpdateSQL(self):
     def FinalizeTestSuiteUpdateSQL(self):
         strQ = self._ReplaceParms(read_template_sql_file("ex_update_test_suite.sql", "execution"))
         return strQ
-    
+
     def CalcPrevalenceTestResultsSQL(self):
         return self._ReplaceParms(read_template_sql_file("ex_calc_prevalence_test_results.sql", "execution"))
 
     def TestScoringRollupRunSQL(self):
         return self._get_rollup_scores_sql().GetRollupScoresTestRunQuery()
-    
+
     def TestScoringRollupTableGroupSQL(self):
         return self._get_rollup_scores_sql().GetRollupScoresTestTableGroupQuery()

pyproject.toml

@@ -53,6 +53,7 @@ def _ReplaceParms(self, strInputString: str):
         strInputString = strInputString.replace("{EXCEPTION_MESSAGE}", self.exception_message)
         strInputString = strInputString.replace("{START_TIME}", self.today)
         strInputString = strInputString.replace("{PROCESS_ID}", str(self.process_id))
+        strInputString = strInputString.replace("{VARCHAR_TYPE}", "STRING" if self.flavor == "databricks" else "VARCHAR")
         strInputString = strInputString.replace(
             "{NOW}", date_service.get_now_as_string_with_offset(self.minutes_offset)
         )
@@ -88,9 +89,9 @@ def _ReplaceParms(self, strInputString: str):
             if parm == "subset_condition":
                 strInputString = strInputString.replace("{SUBSET_DISPLAY}", value.replace("'", "''") if value else "")
 
-
-        # Adding escape character where ':' is referenced
-        strInputString = strInputString.replace(":", "\\:")
+        if self.flavor != "databricks":
+            # Adding escape character where ':' is referenced
+            strInputString = strInputString.replace(":", "\\:")
 
         return strInputString
 

testgen/commands/queries/execute_cat_tests_query.py

@@ -7,6 +7,7 @@
     RunActionQueryList,
     RunThreadedRetrievalQueryList,
     WriteListToDB,
+    date_service,
 )
 
 LOG = logging.getLogger("testgen")
@@ -67,7 +68,11 @@ def FinalizeTestRun(clsCATExecute: CCATExecutionSQL):
                   clsCATExecute.TestScoringRollupRunSQL(),
                   clsCATExecute.TestScoringRollupTableGroupSQL()]
     RunActionQueryList(("DKTG"), lstQueries)
-    run_refresh_score_cards_results(project_code=clsCATExecute.project_code)
+    run_refresh_score_cards_results(
+        project_code=clsCATExecute.project_code,
+        add_history_entry=True,
+        refresh_date=date_service.parse_now(clsCATExecute.run_date),
+    )
 
 
 def run_cat_test_queries(

testgen/commands/queries/execute_tests_query.py

@@ -144,6 +144,7 @@ def run_execution_steps(project_code: str, test_suite: str, minutes_offset: int=
         test_exec_params["connect_by_key"],
         test_exec_params["private_key"],
         test_exec_params["private_key_passphrase"],
+        test_exec_params["http_path"],
         "PROJECT",
     )
 

testgen/commands/run_execute_cat_tests.py

@@ -34,6 +34,7 @@ def run_test_gen_queries(strTableGroupsID, strTestSuite, strGenerationSet=None):
         dctParms["connect_by_key"],
         dctParms["private_key"],
         dctParms["private_key_passphrase"],
+        dctParms["http_path"],
         "PROJECT",
     )