Support find similar search

apps/crawl-to-rag/Dockerfile

@@ -8,8 +8,9 @@ FROM aperturedata/workflows-base
 
 ENV APP_NAME=workflows-crawl-to-rag
 
-# Needed for text-embeddings
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+# Install dependencies for embeddings
+RUN pip install --no-cache-dir -r /app/embeddings/requirements_cpu.txt
+RUN pip install --no-cache-dir -r /app/embeddings/requirements.txt
 
 # copy in the app directories
 COPY --from=crawl-website   /app /workflows/crawl-website
@@ -28,6 +29,7 @@ RUN pip install --no-cache-dir -r /requirements.txt
 COPY --from=rag     /requirements.txt /requirements.txt
 RUN pip install --no-cache-dir -r /requirements.txt
 
+
 EXPOSE 8000
 
 COPY app.sh /app/

apps/rag/Dockerfile

@@ -8,6 +8,10 @@ RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r /requirements.txt
 
+# Install dependencies for embeddings
+RUN pip install --no-cache-dir -r /app/embeddings/requirements_cpu.txt
+RUN pip install --no-cache-dir -r /app/embeddings/requirements.txt
+
 # We prefer to cache models in the docker image rather than load them
 # at run time. 
 COPY app/llm.py /app/llm.py

apps/sql-server/Dockerfile

@@ -5,7 +5,7 @@ ENV APP_NAME=workflows-sql-server
 ENV POSTGRES_VERSION=17
 ARG MULTICORN_VERSION=3.0
 
-# Add PGDG repository and install PostgreSQL 17
+# Add PGDG repository and install PostgreSQL 
 RUN apt-get update && apt-get install -y wget gnupg lsb-release \
  && echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \
       > /etc/apt/sources.list.d/pgdg.list \
@@ -23,28 +23,39 @@ RUN echo "listen_addresses = '*'" >> /etc/postgresql/${POSTGRES_VERSION}/main/po
 RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/17/main/pg_hba.conf
 
 # Postgres/Multicorn insists on using the system Python, so we need to disable the virtual environment
+# Store current VIRTUAL_ENV and PATH values
+ENV OLD_VIRTUAL_ENV="${VIRTUAL_ENV}"
+ENV OLD_PATH="${PATH}"
+ENV OLD_PYTHONPATH="${PYTHONPATH}"
+
+# Disable virtual environment
 ENV VIRTUAL_ENV=
 ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PYTHONPATH=""
 
 # Install multicorn2 Python module into system Python
 RUN /usr/bin/python3 -m pip install --no-cache-dir "git+https://github.com/pgsql-io/multicorn2.git"
 
-# Build and install multicorn2 Postgres extension
-RUN git clone https://github.com/pgsql-io/multicorn2.git /multicorn2 \
+# Build and install multicorn2 Postgres extension 
+RUN git clone  --single-branch --branch main https://github.com/pgsql-io/multicorn2.git /multicorn2 \
  && cd /multicorn2 \
  && make PYTHON=/usr/bin/python3 \
  && make install
 
-RUN pip install --no-cache-dir aperturedb dotenv
+RUN pip install --no-cache-dir aperturedb dotenv numpy pydantic
+
+# Install dependencies for embeddings
+RUN pip install --no-cache-dir -r /app/embeddings/requirements_cpu.txt
+RUN pip install --no-cache-dir -r /app/embeddings/requirements.txt
 
 # Copy and install our FDW into system Python
 COPY fdw /fdw
 RUN cd /fdw && /usr/bin/python3 -m pip install .
 
 # Restore virtual environment
-ENV VIRTUAL_ENV=/opt/venv
-ENV PATH="/opt/venv/bin:/opt/venv/lib/python3.10/site-packages:$PATH"
-ENV PYTHONPATH="/app:/opt/venv/lib/python3.10/site-packages"
+ENV VIRTUAL_ENV=${OLD_VIRTUAL_ENV}
+ENV PATH="${OLD_PATH}"
+ENV PYTHONPATH="/app:${OLD_PYTHONPATH}"
 
 # Install application requirements
 COPY requirements.txt /requirements.txt

apps/sql-server/app/sql/functions.sql

@@ -102,4 +102,43 @@ $$ LANGUAGE SQL IMMUTABLE;
 CREATE OR REPLACE FUNCTION OPERATIONS(VARIADIC ops jsonb[])
 RETURNS jsonb AS $$
   SELECT jsonb_agg(op) FROM unnest($1) AS op
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL IMMUTABLE;
+
+
+-- Find similar
+
+CREATE OR REPLACE FUNCTION FIND_SIMILAR(
+    text TEXT DEFAULT NULL,
+    image BYTEA DEFAULT NULL,
+    vector JSONB DEFAULT NULL,
+    k INT DEFAULT 10,
+    knn_first BOOLEAN DEFAULT TRUE
+) RETURNS JSONB AS $$
+DECLARE
+    mode_count INT;
+BEGIN
+    -- Count how many modes are specified
+    mode_count := (CASE WHEN text IS NOT NULL THEN 1 ELSE 0 END) +
+                  (CASE WHEN image IS NOT NULL THEN 1 ELSE 0 END) +
+                  (CASE WHEN vector IS NOT NULL THEN 1 ELSE 0 END);
+
+    IF mode_count != 1 THEN
+        RAISE EXCEPTION 'FIND_SIMILAR requires exactly one of text, image, or vector';
+    END IF;
+
+    IF k IS NULL OR k <= 0 THEN
+        RAISE EXCEPTION 'k must be a positive integer';
+    END IF;
+
+    RETURN jsonb_build_object(
+        'type', 'find_similar',
+        'text', text,
+        'image', image,
+        'vector', vector,
+        'k_neighbors', k,
+        'knn_first', knn_first
+    );
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION FIND_SIMILAR IS 'Find similar items based on one of text, image, or vector.';