Added VECTOR_DB_INDEX environment variable for overriding the Faiss database index

Author: syedriko

byok/Containerfile.output

@@ -2,12 +2,14 @@ ARG BYOK_TOOL_IMAGE=quay.io/$USERNAME/tool:latest
 ARG UBI_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:latest
 FROM ${BYOK_TOOL_IMAGE} as tool
 ARG UBI_BASE_IMAGE
+ARG VECTOR_DB_INDEX=vector_db_index
 
 USER 0
 WORKDIR /workdir
 
+ENV VECTOR_DB_INDEX=$VECTOR_DB_INDEX
 RUN python3.11 generate_embeddings_tool.py -i /markdown -emd embeddings_model \
-    -emn sentence-transformers/all-mpnet-base-v2 -o vector_db -id vector_db_index
+    -emn sentence-transformers/all-mpnet-base-v2 -o vector_db -id $VECTOR_DB_INDEX
 
 FROM ${UBI_BASE_IMAGE}
 COPY --from=tool /workdir/vector_db /rag/vector_db

byok/Containerfile.tool

@@ -3,6 +3,7 @@ ARG UBI_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:latest
 FROM ${UBI_BASE_IMAGE}
 ARG LOG_LEVEL=info
 ARG OUT_IMAGE_TAG=byok-image
+ARG VECTOR_DB_INDEX=vector_db_index
 ARG BYOK_TOOL_IMAGE
 ARG UBI_BASE_IMAGE
 RUN dnf install -y buildah python3.11 python3.11-pip wget
@@ -25,7 +26,9 @@ ENV OUT_IMAGE_TAG=$OUT_IMAGE_TAG
 ENV BYOK_TOOL_IMAGE=$BYOK_TOOL_IMAGE
 ENV UBI_BASE_IMAGE=$UBI_BASE_IMAGE
 ENV LOG_LEVEL=$LOG_LEVEL
+ENV VECTOR_DB_INDEX=$VECTOR_DB_INDEX
 CMD buildah --log-level $LOG_LEVEL build --build-arg BYOK_TOOL_IMAGE=$BYOK_TOOL_IMAGE \
-    --build-arg UBI_BASE_IMAGE=$UBI_BASE_IMAGE -t $OUT_IMAGE_TAG -f Containerfile.output \
+    --build-arg UBI_BASE_IMAGE=$UBI_BASE_IMAGE --env VECTOR_DB_INDEX=$VECTOR_DB_INDEX \
+    -t $OUT_IMAGE_TAG -f Containerfile.output \
     -v /markdown:/markdown:Z . && rm -f /output/$OUT_IMAGE_TAG.tar && \
     buildah push $OUT_IMAGE_TAG docker-archive:/output/$OUT_IMAGE_TAG.tar

byok/README.md

@@ -27,7 +27,10 @@ $ podman push $MY_BYOK_TOOL_IMAGE
 
 ```bash
 $ MY_BYOK_TOOL_IMAGE=quay.io/$USERNAME/byok_tool:0.0.1
-$ podman run -e OUT_IMAGE_TAG=my-byok-image -it --rm --device=/dev/fuse -v <dir_tree_with_markdown_files>:/markdown:Z -v <dir_for_image_tar>:/output:Z $MY_BYOK_TOOL_IMAGE
+$ podman run -e OUT_IMAGE_TAG=my-byok-image -it --rm --device=/dev/fuse \
+  -v <dir_tree_with_markdown_files>:/markdown:Z \
+  -v <dir_for_image_tar>:/output:Z \
+  $MY_BYOK_TOOL_IMAGE
 ```
 
 The tool runs on CPUs, not GPUs.
@@ -71,4 +74,8 @@ total 1408
 $ podman run --rm localhost/byok-image-foobar:latest cat /rag/vector_db/metadata.json
 {"execution-time": 82.21773672103882, "llm": "None", "embedding-model-name": "sentence-transformers/all-mpnet-base-v2", "index-id": "vector_db_index", "vector-db": "faiss.IndexFlatIP", "embedding-dimension": 768, "chunk": 380, "overlap": 0, "total-embedded-files": 29}
 ```
-The database is located at /rag/vector_db and the Faiss index id is "vector_db_index".
+The database is located at /rag/vector_db and by default the Faiss index id is "vector_db_index". The VECTOR_DB_INDEX environment variable can be used to override this default:
+
+```bash
+$ podman run -e VECTOR_DB_INDEX=acme_openshift_sop ...
+```

byok/registries.conf

@@ -0,0 +1,3 @@
+[[registry]]
+location = "localhost:5000"
+insecure = true