Merge pull request #85 from raptorsun/hermetic-fix

fix: install lightspeed_rag_content module itself in hermetic built image

Author: tisnik

.tekton/rag-tool-pull-request.yaml

@@ -58,12 +58,13 @@ spec:
           "path": ".",
           "requirements_files": [
             "requirements.hashes.wheel.txt",
+            "requirements.hashes.wheel.pypi.txt",
             "requirements.hashes.source.txt",
             "requirements.hermetic.txt"
           ],
           "requirements_build_files": ["requirements-build.txt"],
           "binary": {
-            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling,docling-core,docling-parse,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,lxml,markdown-it-py,markupsafe,mdurl,mpmath,mypy-extensions,nest-asyncio,networkx,nltk,numpy,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pydantic,pydantic-core,pydantic-settings,pygments,pylatexenc,pypdfium2,python-dateutil,python-docx,python-dotenv,python-pptx,pytz,pyyaml,referencing,requests,rpds-py,rtree,safetensors,scikit-learn,scipy,setuptools,shellingham,six,sniffio,starlette,sympy,tabulate,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tornado,transformers,triton,typer,typing-extensions,typing-inspect,typing-inspection,tzdata,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,rapidocr,sqlite-vec",
+            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling-parse,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,lxml,markdown-it-py,markupsafe,mdurl,mpire,mpmath,mypy-extensions,nest-asyncio,networkx,nltk,numpy,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pydantic,pydantic-core,pydantic-settings,pygments,pylatexenc,pypdfium2,python-dateutil,python-docx,python-dotenv,python-pptx,pytz,pyyaml,referencing,requests,rpds-py,rtree,safetensors,scikit-learn,scipy,semchunk,setuptools,shapely,shellingham,six,sniffio,sympy,tabulate,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tornado,transformers,triton,typing-extensions,typing-inspect,typing-inspection,tzdata,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,omegaconf,rapidocr,sqlite-vec,griffe,griffecli,griffelib,pyclipper,tree-sitter-typescript",
             "os": "linux",
             "arch": "x86_64,aarch64",
             "py_version": 312

.tekton/rag-tool-push.yaml

@@ -53,12 +53,13 @@ spec:
           "path": ".",
           "requirements_files": [
             "requirements.hashes.wheel.txt",
+            "requirements.hashes.wheel.pypi.txt",
             "requirements.hashes.source.txt",
             "requirements.hermetic.txt"
           ],
           "requirements_build_files": ["requirements-build.txt"],
           "binary": {
-            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling,docling-core,docling-parse,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,lxml,markdown-it-py,markupsafe,mdurl,mpmath,mypy-extensions,nest-asyncio,networkx,nltk,numpy,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pydantic,pydantic-core,pydantic-settings,pygments,pylatexenc,pypdfium2,python-dateutil,python-docx,python-dotenv,python-pptx,pytz,pyyaml,referencing,requests,rpds-py,rtree,safetensors,scikit-learn,scipy,setuptools,shellingham,six,sniffio,starlette,sympy,tabulate,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tornado,transformers,triton,typer,typing-extensions,typing-inspect,typing-inspection,tzdata,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,rapidocr,sqlite-vec",
+            "packages": "accelerate,aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,attrs,beautifulsoup4,cffi,chardet,charset-normalizer,click,colorama,cryptography,dataclasses-json,defusedxml,distro,docling-parse,et-xmlfile,faiss-cpu,filetype,fire,frozenlist,googleapis-common-protos,greenlet,h11,hf-xet,httpcore,httpx,idna,jinja2,jiter,joblib,jsonlines,jsonref,jsonschema-specifications,latex2mathml,lxml,markdown-it-py,markupsafe,mdurl,mpire,mpmath,mypy-extensions,nest-asyncio,networkx,nltk,numpy,openpyxl,opentelemetry-api,opentelemetry-exporter-otlp-proto-common,opentelemetry-exporter-otlp-proto-http,opentelemetry-proto,opentelemetry-sdk,opentelemetry-semantic-conventions,packaging,pandas,pillow,platformdirs,pluggy,prompt-toolkit,propcache,psycopg2-binary,pydantic,pydantic-core,pydantic-settings,pygments,pylatexenc,pypdfium2,python-dateutil,python-docx,python-dotenv,python-pptx,pytz,pyyaml,referencing,requests,rpds-py,rtree,safetensors,scikit-learn,scipy,semchunk,setuptools,shapely,shellingham,six,sniffio,sympy,tabulate,threadpoolctl,tiktoken,tokenizers,torch,torchvision,tornado,transformers,triton,typing-extensions,typing-inspect,typing-inspection,tzdata,wrapt,xlsxwriter,yarl,zipp,uv-build,uv,pip,maturin,opencv-python,omegaconf,rapidocr,sqlite-vec,griffe,griffecli,griffelib,pyclipper,tree-sitter-typescript",
             "os": "linux",
             "arch": "x86_64,aarch64",
             "py_version": 312

Containerfile

@@ -16,7 +16,7 @@ RUN pip3.12 install uv>=0.7.20
 
 WORKDIR /rag-content
 
-COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock requirements.hashes.wheel.txt requirements.hashes.source.txt requirements-build.txt ./
+COPY Makefile pyproject.toml uv.lock README.md Gemfile Gemfile.lock requirements.hashes.wheel.txt requirements.hashes.wheel.pypi.txt requirements.hashes.source.txt requirements-build.txt ./
 COPY src ./src
 COPY tests ./tests
 COPY scripts ./scripts
@@ -38,7 +38,8 @@ RUN if [ -f /cachi2/cachi2.env ]; then \
     . /cachi2/cachi2.env && \
     uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && \
     . .venv/bin/activate && \
-    pip install --no-cache-dir --ignore-installed --no-index --find-links ${PIP_FIND_LINKS} --no-deps -r requirements.hashes.wheel.txt -r requirements.hashes.source.txt && \
+    pip install --no-cache-dir --ignore-installed --no-index --find-links ${PIP_FIND_LINKS} --no-deps -r requirements.hashes.wheel.txt -r requirements.hashes.wheel.pypi.txt -r requirements.hashes.source.txt && \
+    pip install --no-cache-dir --no-deps . && \
     pip check; \
     else \
     uv sync --locked --no-dev; \

pyproject.toml

@@ -46,11 +46,11 @@ dependencies = [
     "faiss-cpu>=1.11.0.post1",
     "llama-index-vector-stores-postgres>=0.5.4",
     # Pin torch/torchvision to versions available as CPU wheels
-    # torch 2.8.x pairs with torchvision 0.23.x
+    # torch 2.9.x pairs with torchvision 0.24.x
     "llama-stack==0.4.3",
     "llama-stack-client==0.4.3",
-    "torch>=2.8.0,<2.9.0",
-    "torchvision>=0.23.0,<0.24.0",
+    "torch>=2.9.0,<2.10.0",
+    "torchvision>=0.24.0,<0.25.0",
     "aiosqlite>=0.21.0",
     "sqlite-vec>=0.1.6",
     "tomlkit",
@@ -69,6 +69,15 @@ dynamic = ["license", "readme"]
 name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
+# Require wheels for Linux x86_64 so torchvision resolves to a build that has
+# manylinux wheels (the PyTorch CPU index has multiple builds; without this,
+# uv can pick a build that only has win_arm64 and then sync fails on Linux).
+[tool.uv]
+required-environments = [
+    "sys_platform == 'linux' and platform_machine == 'x86_64'",
+    "sys_platform == 'linux' and platform_machine == 'aarch64'",
+]
+
 [tool.uv.sources]
 torch = [{ index = "pytorch-cpu" }]
 torchvision = [{ index = "pytorch-cpu" }]

requirements-build.txt

@@ -61,6 +61,7 @@ hatchling==1.28.0
     #   polyfactory
     #   python-multipart
     #   soupsieve
+    #   starlette
     #   termcolor
     #   urllib3
     #   uvicorn
@@ -77,8 +78,8 @@ pathspec==1.0.4
 pdm-backend==2.4.7
     # via
     #   fastapi
-    #   griffe
     #   marko
+    #   typer
 pluggy==1.6.0
     # via hatchling
 poetry-core==2.3.1
@@ -108,17 +109,23 @@ wheel==0.46.3
     # via
     #   pycparser
     #   sentence-transformers
+    #   tree-sitter-c
+    #   tree-sitter-javascript
+    #   tree-sitter-python
 
 # The following packages are considered to be unsafe in a requirements file:
-setuptools==80.10.2
+setuptools==82.0.0
     # via
     #   calver
     #   certifi
+    #   colorlog
+    #   dill
     #   importlib-metadata
     #   llama-stack
     #   llama-stack-api
     #   maturin
     #   multidict
+    #   multiprocess
     #   pathspec
     #   pgvector
     #   pluggy
@@ -132,4 +139,8 @@ setuptools==80.10.2
     #   sqlalchemy
     #   tenacity
     #   tqdm
+    #   tree-sitter
+    #   tree-sitter-c
+    #   tree-sitter-javascript
+    #   tree-sitter-python
     #   trove-classifiers