Add web crawling (#4)

* Initial implementation of web-crawling.  Each file found is ingested.
  * Leverages crawler4j dependency
* Restore OpenAI interactions that rely on embedding model
* Upgrades to docker-compose infra.
* Add crawl function to ENDPOINTS.md and Excalidraw-based diagram
* Other miscellaneous adjustments

Author: pacphi

build.gradle

@@ -49,12 +49,18 @@ dependencies {
 	}
 	implementation 'io.minio:minio:8.5.12'
 	implementation 'com.emc.ecs:object-client:3.4.10'
-	implementation 'io.pivotal.cfenv:java-cfenv-all:3.2.0'
+	implementation 'io.pivotal.cfenv:java-cfenv:3.2.0'
+	implementation 'io.pivotal.cfenv:java-cfenv-boot:3.2.0'
+	implementation 'com.cedarsoftware:json-io:4.30.0'
 	implementation 'org.springframework.cloud:spring-cloud-bindings:2.0.3'
 	implementation 'org.springframework.ai:spring-ai-markdown-document-reader'
 	implementation 'org.springframework.ai:spring-ai-tika-document-reader'
 	implementation 'org.springframework.ai:spring-ai-pdf-document-reader'
 	implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
+	implementation 'io.github.springboot-addons:spring-boot-starter-httpclient5-resilience4j:1.0.5'
+	implementation ('de.hs-heilbronn.mi:crawler4j-with-hsqldb:5.1.0') {
+        exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl'
+    }
 	runtimeOnly 'io.micrometer:micrometer-registry-prometheus'
 	developmentOnly 'org.springframework.boot:spring-boot-docker-compose'
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'

docker-compose.chroma.yml

@@ -3,7 +3,7 @@ include:
 
 services:
   chroma:
-    image: chromadb/chroma:0.5.5
+    image: chromadb/chroma:0.5.15
     environment:
       - IS_PERSISTENT=TRUE
     volumes:

docker-compose.minio.yml

@@ -1,6 +1,6 @@
 services:
   minio:
-    image: minio/minio:RELEASE.2024-09-22T00-33-43Z.fips
+    image: minio/minio:RELEASE.2024-10-13T13-34-11Z.fips
     ports:
       - 9000:9000
       - 9001:9001

docker-compose.observability.yml

@@ -7,7 +7,7 @@ services:
     ports:
       - 9411:9411
   prometheus:
-    image: prom/prometheus:v2.54.1
+    image: prom/prometheus:v2.55.0
     container_name: prometheus
     volumes:
       - "./prometheus.yml:/etc/prometheus/prometheus.yml"
@@ -16,7 +16,7 @@ services:
     ports:
       - 9090:9090
   grafana:
-    image: grafana/grafana:11.2.0
+    image: grafana/grafana:11.3.0
     container_name: grafana
     restart: unless-stopped
     ports:

docker-compose.redis.yml

@@ -3,7 +3,7 @@ include:
 
 services:
   redis:
-    image: redis/redis-stack-server:7.4.0-v0
+    image: redis/redis-stack-server:7.4.0-v1
     container_name: redis
     ports:
       - 6379:6379

docs/ENDPOINTS.md

@@ -2,6 +2,7 @@
 
 * [Endpoints](#endpoints)
   * [Upload](#upload)
+  * [Crawl](#crawl)
   * [Chat](#chat)
   * [Get Metadata](#get-metadata)
   * [Search](#search)
@@ -11,11 +12,11 @@
 
 ## Endpoints
 
-All endpoints below are to be prefixed with `/api/files`.
+All endpoints with exception to `/crawl` below are to be prefixed with `/api/files`.
 
 ### Upload
 
-Upload a file to an S3-compliant object store's bucket
+Upload a file to an S3-compliant object store's bucket; in addition the contents of the file will be vectorized and persisted into a [Vector Database](https://docs.spring.io/spring-ai/reference/api/vectordbs.html) provider.
 
 ```python
 POST /upload
@@ -56,6 +57,32 @@ You will need to adjust startup arguments, e.g., you could add the following to
 -Dspring.servlet.multipart.max-file-size=250MB
 ```
 
+### Crawl
+
+Facilitate web-crawling.  Users may issue a [CrawlRequest](../src/main/java/org/cftoolsuite/domain/crawl/CrawlRequest.java) and each document found will be (a) uploaded to S3-compliant object store and (b) contents will be vectorized and persisted into a Vector Database provider.
+
+```python
+POST /crawl
+```
+
+**Sample interaction**
+
+```bash
+❯ http POST :8080/crawl rootDomain="https://docs.vmware.com/en/VMware-Tanzu-Platform/SaaS/" seeds:='["https://docs.vmware.com/en/VMware-Tanzu-Platform/SaaS/create-manage-apps-tanzu-platform-k8s/"]' maxDepthOfCrawling=5
+HTTP/1.1 202
+Connection: keep-alive
+Content-Type: application/json
+Date: Thu, 24 Oct 2024 13:38:24 GMT
+Keep-Alive: timeout=60
+Transfer-Encoding: chunked
+
+{
+    "id": "1",
+    "result": "Accepted",
+    "storageFolder": "/tmp/crawler4j/1"
+}
+```
+
 ### Chat
 
 Converse with an AI chatbot who is aware of all uploaded content.  Ask a question, get a response.
@@ -80,7 +107,6 @@ Hermia is a key character in "A Midsummer Night's Dream," portrayed as the daugh
 She passionately defends her love for Lysander and is determined to be with him despite the obstacles posed by her father and societal expectations. Hermia's character embodies themes of love, rebellion, and the struggle for autonomy within the constraints of Athenian law. Her determination to follow her heart leads her to plan an escape with Lysander, showcasing her bravery and commitment to love (Act I).
 ```
 
-
 ### Get Metadata
 
 Retrieve metadata for all files in an S3-compliant object store's bucket