Add support for Elasticsearch vector store

.github/workflows/continuous-integration.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Generate Java docs
         run: mvn javadoc:aggregate
 
-      - name: Generate Reference docs
+      - name: Generate assembly
         working-directory: spring-ai-docs
         run: mvn assembly:single
 
@@ -60,5 +60,5 @@ jobs:
         working-directory: spring-ai-docs/target
         run: |
           unzip spring-ai-$PROJECT_VERSION-docs.zip
-          ssh -i $HOME/.ssh/key $DOCS_USERNAME@$DOCS_HOST cd $DOCS_PATH && mkdir -p $PROJECT_VERSION/api
+          ssh -i $HOME/.ssh/key $DOCS_USERNAME@$DOCS_HOST "cd $DOCS_PATH && mkdir -p $PROJECT_VERSION/api"
           scp -i $HOME/.ssh/key -r api $DOCS_USERNAME@$DOCS_HOST:$DOCS_PATH/$PROJECT_VERSION/api

.github/workflows/documentation-upload.yml

@@ -28,9 +28,9 @@ jobs:
       - name: Generate Java docs
         run: mvn javadoc:aggregate
 
-      - name: Generate Reference docs
+      - name: Generate assembly
         working-directory: spring-ai-docs
-        run: mvn site
+        run: mvn assembly:single
 
       - name: Setup SSH key
         env:
@@ -50,7 +50,6 @@ jobs:
         working-directory: spring-ai-docs/target
         run: |
           unzip spring-ai-$RELEASE_VERSION-docs.zip
-          ssh -i $HOME/.ssh/key $DOCS_USERNAME@$DOCS_HOST cd $DOCS_PATH && mkdir $RELEASE_VERSION
-          scp -i $HOME/.ssh/key -r api reference $DOCS_USERNAME@$DOCS_HOST:$DOCS_PATH/$RELEASE_VERSION
-          ssh -i $HOME/.ssh/key $DOCS_USERNAME@$DOCS_HOST rm current && ln -s $RELEASE_VERSION current
-
+          ssh -i $HOME/.ssh/key $DOCS_USERNAME@$DOCS_HOST "cd $DOCS_PATH && mkdir -p $RELEASE_VERSION/api"
+          scp -i $HOME/.ssh/key -r api $DOCS_USERNAME@$DOCS_HOST:$DOCS_PATH/$RELEASE_VERSION
+

.gitignore

@@ -34,4 +34,6 @@ node_modules
 package-lock.json
 package.json
 .vscode
-.antlr
+.antlr
+
+shell.log

README.md

@@ -6,7 +6,7 @@ The Spring AI project provides a Spring-friendly API and abstractions for develo
 
 Let's make your `@Beans` intelligent!
 
-For further information go to our [Spring AI refernce documentation](https://docs.spring.io/spring-ai/reference/).
+For further information go to our [Spring AI reference documentation](https://docs.spring.io/spring-ai/reference/).
 
 ## Project Links
 
@@ -17,7 +17,7 @@ For further information go to our [Spring AI refernce documentation](https://doc
 
 ## Educational Resources
 
-- Follow the [Workshop](#workshop)
+- Follow the [Workshop material for Azure OpenAI](https://github.com/Azure-Samples/spring-ai-azure-workshop)
   - The workshop contains step-by-step examples from 'hello world' to 'retrieval augmented generation'
 
 Some selected videos.  Search YouTube! for more.
@@ -29,19 +29,6 @@ Some selected videos.  Search YouTube! for more.
 * Introducing Spring AI - Add Generative AI to your Spring Applications
 <br>[![Watch the video](https://img.youtube.com/vi/1g_wuincUdU/default.jpg)](https://www.youtube.com/watch?v=1g_wuincUdU)
 
-
-## Workshop
-
-* You can try out the features of Spring AI by following the [workshop material for Azure OpenAI](https://github.com/Azure-Samples/spring-ai-azure-workshop)
-* To use the workshop material with OpenAI (not Azure's offering) you will need to *replace* the Azure Open AI Boot Starter in the `pom.xml` with the Open AI Boot Starter.
-```xml
-    <dependency>
-        <groupId>org.springframework.ai</groupId>
-        <artifactId>spring-ai-openai-spring-boot-starter</artifactId>
-        <version>0.7.1-SNAPSHOT</version>
-    </dependency>
-```
-
 ## Getting Started
 
 Please refer to the [Getting Started Guide](https://docs.spring.io/spring-ai/reference/getting-started.html) for instruction on adding your dependencies.
@@ -96,7 +83,7 @@ You can find more details in the [Reference Documentation](https://docs.spring.i
 
 Spring AI supports many AI models.  For an overview see here.  Specific models currently supported are
 * OpenAI
-* 
+*
 
 
 **Prompts:** Central to AI model interaction is the Prompt, which provides specific instructions for the AI to act upon.
@@ -202,3 +189,17 @@ To build the docs
 
 The docs are then in the directory `spring-ai-docs/target/antora/site/index.html`
 
+To reformat using the [java-format plugin](https://github.com/spring-io/spring-javaformat)
+```shell
+./mvnw spring-javaformat:apply
+```
+
+To update the year on license headers using the [license-maven-plugin](https://oss.carbou.me/license-maven-plugin/#goals)
+```shell
+./mvnw license:update-file-header -Plicense
+```
+
+To check javadocs using the [javadoc:javadoc](https://maven.apache.org/plugins/maven-javadoc-plugin/)
+```shell
+./mvnw javadoc:javadoc -Pjavadoc
+```

document-readers/pdf-reader/pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.springframework.ai</groupId>
 		<artifactId>spring-ai</artifactId>
-		<version>0.8.1-SNAPSHOT</version>
+		<version>1.0.0-SNAPSHOT</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<artifactId>spring-ai-pdf-document-reader</artifactId>

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/PagePdfDocumentReader.java

@@ -1,19 +1,18 @@
 /*
- * Copyright 2023-2023 the original author or authors.
+ * Copyright 2023 - 2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      https://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.ai.reader.pdf;
 
 import java.awt.Rectangle;
@@ -26,6 +25,8 @@
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.ai.document.Document;
 import org.springframework.ai.document.DocumentReader;
 import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
@@ -45,6 +46,8 @@
  */
 public class PagePdfDocumentReader implements DocumentReader {
 
+	private final Logger logger = LoggerFactory.getLogger(getClass());
+
 	private static final String PDF_PAGE_REGION = "pdfPageRegion";
 
 	public static final String METADATA_START_PAGE_NUMBER = "page_number";
@@ -100,7 +103,17 @@ public List<Document> get() {
 
 			List<String> pageTextGroupList = new ArrayList<>();
 
+			int totalPages = this.document.getDocumentCatalog().getPages().getCount();
+			int logFrequency = totalPages > 10 ? totalPages / 10 : 1; // if less than 10
+																		// pages, print
+																		// each iteration
+			int counter = 0;
+
 			for (PDPage page : this.document.getDocumentCatalog().getPages()) {
+				if (counter % logFrequency == 0 && counter / logFrequency < 10) {
+					logger.info("Processing PDF page: {}", (counter + 1));
+				}
+				counter++;
 
 				pagesPerDocument++;
 
@@ -140,7 +153,7 @@ public List<Document> get() {
 				readDocuments.add(toDocument(pageTextGroupList.stream().collect(Collectors.joining()), startPageNumber,
 						pageNumber));
 			}
-
+			logger.info("Processing {} pages", totalPages);
 			return readDocuments;
 
 		}

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/ParagraphPdfDocumentReader.java

@@ -1,19 +1,18 @@
 /*
- * Copyright 2023-2023 the original author or authors.
+ * Copyright 2023 - 2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      https://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.ai.reader.pdf;
 
 import java.awt.Rectangle;
@@ -24,6 +23,8 @@
 import org.apache.pdfbox.pdfparser.PDFParser;
 import org.apache.pdfbox.pdmodel.PDDocument;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.ai.document.Document;
 import org.springframework.ai.document.DocumentReader;
 import org.springframework.ai.reader.pdf.config.ParagraphManager;
@@ -47,6 +48,8 @@
  */
 public class ParagraphPdfDocumentReader implements DocumentReader {
 
+	private final Logger logger = LoggerFactory.getLogger(getClass());
+
 	// Constants for metadata keys
 	private static final String METADATA_START_PAGE = "page_number";
 
@@ -129,6 +132,7 @@ public List<Document> get() {
 		List<Document> documents = new ArrayList<>(paragraphs.size());
 
 		if (!CollectionUtils.isEmpty(paragraphs)) {
+			logger.info("Start processing paragraphs from PDF");
 			Iterator<Paragraph> itr = paragraphs.iterator();
 
 			var current = itr.next();
@@ -147,7 +151,7 @@ public List<Document> get() {
 				}
 			}
 		}
-
+		logger.info("End processing paragraphs from PDF");
 		return documents;
 	}
 

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/aot/PdfReaderRuntimeHints.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.springframework.ai.reader.pdf.aot;
 
 import org.springframework.aot.hint.RuntimeHints;

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/ParagraphManager.java

@@ -1,19 +1,18 @@
 /*
- * Copyright 2023-2023 the original author or authors.
+ * Copyright 2023 - 2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      https://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.ai.reader.pdf.config;
 
 import java.io.IOException;

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/config/PdfDocumentReaderConfig.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.springframework.ai.reader.pdf.config;
 
 import org.springframework.ai.reader.ExtractedTextFormatter;

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/layout/ForkPDFLayoutTextStripper.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright 2023 - 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 /* Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

document-readers/pdf-reader/src/main/java/org/springframework/ai/reader/pdf/layout/PDFLayoutTextStripperByArea.java

@@ -1,19 +1,18 @@
 /*
- * Copyright 2023-2023 the original author or authors.
+ * Copyright 2023 - 2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      https://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.ai.reader.pdf.layout;
 
 import java.awt.geom.Rectangle2D;

document-readers/pdf-reader/src/test/java/org/springframework/ai/reader/pdf/PagePdfDocumentReaderTests.java

@@ -1,19 +1,18 @@
 /*
- * Copyright 2023-2024 the original author or authors.
+ * Copyright 2023 - 2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      https://www.apache.org/licenses/LICENSE-2.0
+ * https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.springframework.ai.reader.pdf;
 
 import java.util.List;
@@ -51,7 +50,7 @@ public void classpathRead() {
 
 		assertThat(docs).hasSize(4);
 
-		String allText = docs.stream().map(d -> d.getContent()).collect(Collectors.joining("\n"));
+		String allText = docs.stream().map(d -> d.getContent()).collect(Collectors.joining(System.lineSeparator()));
 
 		assertThat(allText).doesNotContain(
 				List.of("Page  1 of 4", "Page  2 of 4", "Page  3 of 4", "Page  4 of 4", "PDF  Bookmark   Sample"));