Skip to content

Commit 5a04912

Browse files
paulzpaulz
authored
add S3 compatible storage (#6)
Co-authored-by: Paul Zabelin <[email protected]>
1 parent aae414f commit 5a04912

File tree

7 files changed

+1155
-10
lines changed

7 files changed

+1155
-10
lines changed

‎.github/workflows/cat-test-examples.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@ jobs:
3636
env:
3737
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
3838

39+
# - name: Upload artifacts to MinIO
40+
# run: |
41+
# zip -r test-output-${{ github.run_number }}.zip examples/team_recommender/tests/test_runs
42+
# curl -X PUT -T "/path/to/yourfile.zip" \
43+
# -H "Host: localhost:9000" \
44+
# -H "Date: $(date -R)" \
45+
# -H "Content-Type: application/zip" \
46+
# -H "Authorization: AWS minioadmin:minioadmin" \
47+
# http://localhost:9000/yourbucket/yourfile.zip
48+
3949
- name: Upload artifacts to Google Drive
4050
if: always()
4151
run: |

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,3 +317,4 @@ ENV/
317317
# Test artifacts
318318

319319
test_runs/
320+
data-store/

‎data-store/docker-compose.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
version: '3.8'
2+
3+
services:
4+
minio:
5+
image: quay.io/minio/minio
6+
container_name: minio
7+
ports:
8+
- "9000:9000"
9+
- "9001:9001"
10+
environment:
11+
MINIO_ROOT_USER: minioadmin
12+
MINIO_ROOT_PASSWORD: minioadmin
13+
volumes:
14+
- ./minio_data:/data
15+
command: server /data --console-address ":9001"

‎data-store/readme.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
## Example of posting data
2+
3+
4+
```bash
5+
curl -X PUT -T "sample.zip" \
6+
-H "Host: localhost:9000" \
7+
-H "Date: $(date -R)" \
8+
-H "Content-Type: application/zip" \
9+
-H "Authorization: AWS minioadmin:minioadmin" \
10+
http://localhost:9000/test-results/sample.zip
11+
```
12+
13+
if you get `SignatureDoesNotMatch` error, adjust bucket policy to allow public access to the bucket.
14+
15+
16+
## Install client
17+
18+
on macOS:
19+
20+
```bash
21+
brew install minio/stable/mc
22+
mc alias set myminio http://localhost:9000 minioadmin minioadmin
23+
24+
Added `myminio` successfully.
25+
```
26+
27+
copy files to minio:
28+
29+
```bash
30+
mc cp sample.zip myminio/test-results
31+
32+
...gnment-testing/sample.zip: 14.09 KiB / 14.09 KiB ▓▓▓▓▓▓▓
33+
```

‎data-store/reliability-graph.ipynb‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"id": "initial_id",
6+
"metadata": {
7+
"collapsed": true,
8+
"ExecuteTime": {
9+
"end_time": "2025-02-28T23:38:56.420160Z",
10+
"start_time": "2025-02-28T23:38:55.221431Z"
11+
}
12+
},
13+
"source": "!pip install boto3",
14+
"outputs": [
15+
{
16+
"name": "stdout",
17+
"output_type": "stream",
18+
"text": [
19+
"\u001B[33mDEPRECATION: Loading egg at /opt/homebrew/lib/python3.11/site-packages/BlinkStick-1.2.0-py3.11.egg is deprecated. pip 24.3 will enforce this behaviour change. A possible replacement is to use pip for package installation.. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001B[0m\u001B[33m\r\n",
20+
"\u001B[0m\u001B[33mDEPRECATION: Loading egg at /opt/homebrew/lib/python3.11/site-packages/pyusb-1.2.1-py3.11.egg is deprecated. pip 24.3 will enforce this behaviour change. A possible replacement is to use pip for package installation.. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001B[0m\u001B[33m\r\n",
21+
"\u001B[0mRequirement already satisfied: boto3 in /opt/homebrew/lib/python3.11/site-packages (1.37.4)\r\n",
22+
"Requirement already satisfied: botocore<1.38.0,>=1.37.4 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (1.37.4)\r\n",
23+
"Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (1.0.1)\r\n",
24+
"Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in /opt/homebrew/lib/python3.11/site-packages (from boto3) (0.11.3)\r\n",
25+
"Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/artium/Library/Python/3.11/lib/python/site-packages (from botocore<1.38.0,>=1.37.4->boto3) (2.8.2)\r\n",
26+
"Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/homebrew/lib/python3.11/site-packages (from botocore<1.38.0,>=1.37.4->boto3) (1.26.15)\r\n",
27+
"Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.38.0,>=1.37.4->boto3) (1.17.0)\r\n",
28+
"\r\n",
29+
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m24.0\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m25.0.1\u001B[0m\r\n",
30+
"\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpython3.11 -m pip install --upgrade pip\u001B[0m\r\n"
31+
]
32+
}
33+
],
34+
"execution_count": 3
35+
},
36+
{
37+
"metadata": {
38+
"ExecuteTime": {
39+
"end_time": "2025-02-28T23:38:56.444149Z",
40+
"start_time": "2025-02-28T23:38:56.426837Z"
41+
}
42+
},
43+
"cell_type": "code",
44+
"source": [
45+
"import boto3\n",
46+
"from botocore.client import Config\n",
47+
"\n",
48+
"# MinIO server information\n",
49+
"minio_endpoint = 'http://localhost:9000' # Update with your MinIO server URL\n",
50+
"access_key = 'minioadmin' # Replace with your MinIO access key\n",
51+
"secret_key = 'minioadmin' # Replace with your MinIO secret key\n",
52+
"bucket_name = 'your-bucket-name' # Replace with your bucket name\n",
53+
"\n",
54+
"# Create a session and S3 client\n",
55+
"session = boto3.session.Session()\n",
56+
"s3_client = session.client(\n",
57+
" 's3',\n",
58+
" endpoint_url=minio_endpoint,\n",
59+
" aws_access_key_id=access_key,\n",
60+
" aws_secret_access_key=secret_key,\n",
61+
" config=Config(signature_version='s3v4'),\n",
62+
" region_name='us-east-1' # Adjust if necessary\n",
63+
")\n",
64+
"\n",
65+
"# List objects in the specified bucket\n",
66+
"response = s3_client.list_objects_v2(Bucket=bucket_name)\n",
67+
"\n",
68+
"if 'Contents' in response:\n",
69+
" for obj in response['Contents']:\n",
70+
" print(f\"Object: {obj['Key']} (Size: {obj['Size']} bytes)\")\n",
71+
"else:\n",
72+
" print(f\"No objects found in bucket '{bucket_name}'.\")"
73+
],
74+
"id": "708f9b0198a5c3f0",
75+
"outputs": [
76+
{
77+
"ename": "ModuleNotFoundError",
78+
"evalue": "No module named 'boto3'",
79+
"output_type": "error",
80+
"traceback": [
81+
"\u001B[31m---------------------------------------------------------------------------\u001B[39m",
82+
"\u001B[31mModuleNotFoundError\u001B[39m Traceback (most recent call last)",
83+
"\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[4]\u001B[39m\u001B[32m, line 1\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m1\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mboto3\u001B[39;00m\n\u001B[32m 2\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mbotocore\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01mclient\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Config\n\u001B[32m 4\u001B[39m \u001B[38;5;66;03m# MinIO server information\u001B[39;00m\n",
84+
"\u001B[31mModuleNotFoundError\u001B[39m: No module named 'boto3'"
85+
]
86+
}
87+
],
88+
"execution_count": 4
89+
}
90+
],
91+
"metadata": {
92+
"kernelspec": {
93+
"display_name": "Python 3",
94+
"language": "python",
95+
"name": "python3"
96+
},
97+
"language_info": {
98+
"codemirror_mode": {
99+
"name": "ipython",
100+
"version": 2
101+
},
102+
"file_extension": ".py",
103+
"mimetype": "text/x-python",
104+
"name": "python",
105+
"nbconvert_exporter": "python",
106+
"pygments_lexer": "ipython2",
107+
"version": "2.7.6"
108+
}
109+
},
110+
"nbformat": 4,
111+
"nbformat_minor": 5
112+
}

‎pyproject.toml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ authors = [
1212
]
1313
requires-python = "~=3.13"
1414
readme = "README.md"
15-
dependencies = ["pydantic>=2.10.6,<3", "pydrive2>=1.21.3,<2"]
15+
dependencies = [
16+
# this small library should be kept independent
17+
# consider adding dependencies to on of the dependency groups
18+
]
1619
packages = [{ include = "cat_ai", from = "src" }]
1720
license = "MIT"
1821

@@ -28,6 +31,9 @@ dev = [
2831
"sphinx>=8.1.3,<9",
2932
"sphinx-rtd-theme>=3.0.2,<4",
3033
"sphinx-markdown-builder>=0.6.8,<0.7",
34+
"notebook>=7.3.2",
35+
"pydrive2>=1.21.3,<2",
36+
"pydantic>=2.10.6,<3",
3137
]
3238

3339
[tool.uv]
@@ -63,6 +69,6 @@ include = '\.pyi?$'
6369
# Rule selection and configuration
6470
line-length = 120
6571
lint.extend-ignore = ["E203"]
66-
exclude = [".git", "__pycache__", "build", "dist"]
6772
# Equivalent to max-complexity
6873
lint.mccabe = { max-complexity = 10 }
74+
exclude = [".git", "__pycache__", "build", "dist"]

0 commit comments

Comments
 (0)