add S3 compatible storage

.github/workflows/cat-test-examples.yml

@@ -36,6 +36,16 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
+#      - name: Upload artifacts to MinIO
+#        run: |
+#          zip -r test-output-${{ github.run_number }}.zip examples/team_recommender/tests/test_runs
+#            curl -X PUT -T "/path/to/yourfile.zip" \
+#            -H "Host: localhost:9000" \
+#            -H "Date: $(date -R)" \
+#            -H "Content-Type: application/zip" \
+#            -H "Authorization: AWS minioadmin:minioadmin" \
+#            http://localhost:9000/yourbucket/yourfile.zip
+
       - name: Upload artifacts to Google Drive
         if: always()
         run: |

.gitignore

@@ -317,3 +317,4 @@ ENV/
 # Test artifacts
 
 test_runs/
+data-store/

data-store/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  minio:
+    image: quay.io/minio/minio
+    container_name: minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - ./minio_data:/data
+    command: server /data --console-address ":9001"

data-store/readme.md

@@ -0,0 +1,33 @@
+## Example of posting data
+
+
+```bash
+curl -X PUT -T "sample.zip" \
+    -H "Host: localhost:9000" \
+    -H "Date: $(date -R)" \
+    -H "Content-Type: application/zip" \
+    -H "Authorization: AWS minioadmin:minioadmin" \
+    http://localhost:9000/test-results/sample.zip
+```
+
+if you get `SignatureDoesNotMatch` error, adjust bucket policy to allow public access to the bucket.
+
+
+## Install client
+
+on macOS:
+
+```bash
+brew install minio/stable/mc
+mc alias set myminio http://localhost:9000 minioadmin minioadmin
+
+Added `myminio` successfully.
+```
+
+copy files to minio:
+
+```bash
+mc cp sample.zip myminio/test-results
+
+...gnment-testing/sample.zip: 14.09 KiB / 14.09 KiB  ▓▓▓▓▓▓▓
+```

data-store/reliability-graph.ipynb

@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-02-28T23:38:56.420160Z",
+     "start_time": "2025-02-28T23:38:55.221431Z"
+    }
+   },
+   "source": "!pip install boto3",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001B[33mDEPRECATION: Loading egg at /opt/homebrew/lib/python3.11/site-packages/BlinkStick-1.2.0-py3.11.egg is deprecated. pip 24.3 will enforce this behaviour change. A possible replacement is to use pip for package installation.. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001B[0m\u001B[33m\r\n",
+      "\u001B[0m\u001B[33mDEPRECATION: Loading egg at /opt/homebrew/lib/python3.11/site-packages/pyusb-1.2.1-py3.11.egg is deprecated. pip 24.3 will enforce this behaviour change. A possible replacement is to use pip for package installation.. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001B[0m\u001B[33m\r\n",
+      "\u001B[0mRequirement already satisfied: boto3 in /opt/homebrew/lib/python3.11/site-packages (1.37.4)\r\n",
+      "Requirement already satisfied: botocore<1.38.0,>=1.37.4 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (1.37.4)\r\n",
+      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (1.0.1)\r\n",
+      "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (0.11.3)\r\n",
+      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/artium/Library/Python/3.11/lib/python/site-packages (from botocore<1.38.0,>=1.37.4->boto3) (2.8.2)\r\n",
+      "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/homebrew/lib/python3.11/site-packages (from botocore<1.38.0,>=1.37.4->boto3) (1.26.15)\r\n",
+      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.38.0,>=1.37.4->boto3) (1.17.0)\r\n",
+      "\r\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m24.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m25.0.1\u001B[0m\r\n",
+      "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3.11 -m pip install --upgrade pip\u001B[0m\r\n"
+     ]
+    }
+   ],
+   "execution_count": 3
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-02-28T23:38:56.444149Z",
+     "start_time": "2025-02-28T23:38:56.426837Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import boto3\n",
+    "from botocore.client import Config\n",
+    "\n",
+    "# MinIO server information\n",
+    "minio_endpoint = 'http://localhost:9000'  # Update with your MinIO server URL\n",
+    "access_key = 'minioadmin'                 # Replace with your MinIO access key\n",
+    "secret_key = 'minioadmin'                 # Replace with your MinIO secret key\n",
+    "bucket_name = 'your-bucket-name'          # Replace with your bucket name\n",
+    "\n",
+    "# Create a session and S3 client\n",
+    "session = boto3.session.Session()\n",
+    "s3_client = session.client(\n",
+    "    's3',\n",
+    "    endpoint_url=minio_endpoint,\n",
+    "    aws_access_key_id=access_key,\n",
+    "    aws_secret_access_key=secret_key,\n",
+    "    config=Config(signature_version='s3v4'),\n",
+    "    region_name='us-east-1'  # Adjust if necessary\n",
+    ")\n",
+    "\n",
+    "# List objects in the specified bucket\n",
+    "response = s3_client.list_objects_v2(Bucket=bucket_name)\n",
+    "\n",
+    "if 'Contents' in response:\n",
+    "    for obj in response['Contents']:\n",
+    "        print(f\"Object: {obj['Key']} (Size: {obj['Size']} bytes)\")\n",
+    "else:\n",
+    "    print(f\"No objects found in bucket '{bucket_name}'.\")"
+   ],
+   "id": "708f9b0198a5c3f0",
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'boto3'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mModuleNotFoundError\u001B[39m                       Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[4]\u001B[39m\u001B[32m, line 1\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m1\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mboto3\u001B[39;00m\n\u001B[32m      2\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mbotocore\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01mclient\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Config\n\u001B[32m      4\u001B[39m \u001B[38;5;66;03m# MinIO server information\u001B[39;00m\n",
+      "\u001B[31mModuleNotFoundError\u001B[39m: No module named 'boto3'"
+     ]
+    }
+   ],
+   "execution_count": 4
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -12,7 +12,10 @@ authors = [
 ]
 requires-python = "~=3.13"
 readme = "README.md"
-dependencies = ["pydantic>=2.10.6,<3", "pydrive2>=1.21.3,<2"]
+dependencies = [
+    # this small library should be kept independent
+    # consider adding dependencies to on of the dependency groups
+]
 packages = [{ include = "cat_ai", from = "src" }]
 license = "MIT"
 
@@ -28,6 +31,9 @@ dev = [
   "sphinx>=8.1.3,<9",
   "sphinx-rtd-theme>=3.0.2,<4",
   "sphinx-markdown-builder>=0.6.8,<0.7",
+  "notebook>=7.3.2",
+  "pydrive2>=1.21.3,<2",
+  "pydantic>=2.10.6,<3",
 ]
 
 [tool.uv]
@@ -63,6 +69,6 @@ include = '\.pyi?$'
 # Rule selection and configuration
 line-length = 120
 lint.extend-ignore = ["E203"]
-exclude = [".git", "__pycache__", "build", "dist"]
 # Equivalent to max-complexity
 lint.mccabe = { max-complexity = 10 }
+exclude = [".git", "__pycache__", "build", "dist"]