feat/add singlestore source connector (#197)

* add singlestore source connector

* pull all needed docker images at the beginning of the CI job

* Add docker logs to error output

* delete images after docker compose

* shell tidy

* bump the github runner for src/dest integration tests

* fix minio docker compose path

* don't prematurely pull all images

* Fix changelog typo

* replace deprecated __fields__ with model_fields()

Author: rbiseck3

.github/workflows/e2e.yml

@@ -69,7 +69,7 @@ jobs:
           make integration-test-embedders
 
   source_connectors_integration_test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     needs: [ setup ]
     steps:
     - uses: 'actions/checkout@v4'
@@ -84,6 +84,10 @@ jobs:
       uses: ./.github/actions/base-cache
       with:
         python-version: "3.10"
+    - name: Setup up docker
+      run: |
+        sudo make install-docker-compose
+        docker compose version
     - name: Run Integration Tests
       env:
         DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
@@ -92,13 +96,11 @@ jobs:
         DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}}
       run : |
         source .venv/bin/activate
-        sudo make install-docker-compose
-        docker compose version
         make install-test
         make integration-test-connectors-src
 
   destination_connectors_integration_test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     needs: [ setup ]
     steps:
     - uses: 'actions/checkout@v4'
@@ -113,6 +115,10 @@ jobs:
       uses: ./.github/actions/base-cache
       with:
         python-version: "3.10"
+    - name: Setup up docker
+      run: |
+        sudo make install-docker-compose
+        docker compose version
     - name: Run Integration Tests
       env:
         DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
@@ -123,8 +129,6 @@ jobs:
         S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }}
       run : |
         source .venv/bin/activate
-        sudo make install-docker-compose
-        docker compose version
         make install-test
         make integration-test-connectors-dest
 

CHANGELOG.md

@@ -141,6 +141,7 @@ integration-test-embedders:
 integration-test-connectors-src:
 	PYTHONPATH=. pytest --tags source -sv test/integration/connectors
 
+
 .PHONY: integration-test-connectors-dest
 integration-test-connectors-dest:
 	PYTHONPATH=. pytest --tags destination -sv test/integration/connectors

Makefile

@@ -11,11 +11,3 @@ services:
       - ROOT_PASSWORD=password
     volumes:
       - ./schema.sql:/init.sql
-
-  # Allow docker compose up --wait to exit only when singlestore is healthy
-  wait:
-    image: hello-world:latest
-    container_name: singlestore-waiter
-    depends_on:
-      singlestore:
-        condition: service_healthy

test/integration/connectors/env_setup/minio/.env renamed to test/integration/connectors/env_setup/minio/source/.env

@@ -2,7 +2,7 @@ CREATE DATABASE ingest_test;
 USE ingest_test;
 
 CREATE TABLE elements (
-    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    id TEXT PRIMARY KEY NOT NULL,
     element_id TEXT,
     text TEXT,
     embeddings Vector(384),