github tests

Author: saul-data

.github/workflows/python-tests.yaml

@@ -11,18 +11,31 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v3
+
+      - name: Build the stack
+        run: docker-compose -f tests/docker-compose.yaml up --build -d
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          if [ -f .devcontainer/requirements.txt ]; then pip install -r .devcontainer/requirements.txt --use-pep517; fi
+          if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
       - name: Test with pytest
         run: |
-          pytest -s
+          pytest -s
+        env: 
+          SHAREPOINT_HOST: ${{ secrets.SHAREPOINT_HOST }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          TEAMS_WEBHOOK: ${{ secrets.TEAMS_WEBHOOK }}
+          REDIS_HOST: "localhost"
+          S3_HOST: "http://localhost:9000"

src/dataplane/pipelinerun/data_persist/test_redis.py

@@ -5,10 +5,15 @@
 import redis
 from datetime import timedelta
 from nanoid import generate
+from dotenv import load_dotenv
 
 def test_redis_store():
 
+    load_dotenv()
+
     # ---------- Dataplane pipeline run ------------
+    REDIS_HOST = os.environ["REDIS_HOST"]
+    print("Redis:", REDIS_HOST)
 
     # Dataplane run id
     os.environ["DP_RUNID"] = generate('1234567890abcdef', 10)
@@ -23,7 +28,7 @@ def test_redis_store():
     dfrows = df.shape[0]
 
     # Redis connection
-    redisConnect = redis.Redis(host='redis-service', port=6379, db=0)
+    redisConnect = redis.Redis(host=REDIS_HOST, port=6379, db=0)
 
 
     # ---------- STORE PARQUET TO REDIS ------------

src/dataplane/pipelinerun/data_persist/test_s3_store.py

@@ -5,9 +5,12 @@
 import boto3
 from botocore.client import Config
 from nanoid import generate
+from dotenv import load_dotenv
 
 def test_s3_store():
 
+    load_dotenv()
+
     # ---------- DATAPLANE RUN ------------
 
     # Dataplane run id
@@ -22,10 +25,12 @@ def test_s3_store():
     df = pd.DataFrame(data)
     dfrows = df.shape[0]
 
+    S3_HOST = os.environ["S3_HOST"]
+
     # S3 connection
     S3Connect = boto3.client(
         's3',
-        endpoint_url="http://minio:9000",
+        endpoint_url=S3_HOST,
         aws_access_key_id="admin",
         aws_secret_access_key="hello123",
         config=Config(signature_version='s3v4'),

tests/docker-compose.yaml

@@ -0,0 +1,41 @@
+version: '3'
+
+services:
+
+  minio:
+    command: server /data --console-address ":9001"
+    image: minio/minio:RELEASE.2022-10-15T19-57-03Z
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      MINIO_ROOT_USER: admin
+      MINIO_ROOT_PASSWORD: hello123
+    volumes:
+      - miniovolume:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+  
+  createbuckets:
+    image: minio/mc:RELEASE.2022-10-12T18-12-50Z
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9000 admin hello123;
+      /usr/bin/mc mb myminio/dataplanebucket;
+      # /usr/bin/mc policy set public myminio/dataplanebucket;
+      exit 0;
+      "
+
+  redis-service:
+    image: redis:7.0.5-bullseye
+    ports:
+      - 6379:6379
+
+volumes:
+  miniovolume:
+  

tests/requirements.txt

@@ -0,0 +1,10 @@
+# utils==1.0.1
+# docstring==0.1.2.4
+redis==4.3.4
+pandas==1.5.1
+boto3==1.24.93
+pytest==7.1.3
+pyarrow==9.0.0
+nanoid==0.1
+requests==2.28.1
+python-dotenv==0.21.0