Skip to content

Commit 225b13d

Browse files
dantelmomsftdantelmomsft
committed
first working deployment on aca
1 parent c991a0d commit 225b13d

File tree

77 files changed

+744
-366
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+744
-366
lines changed

‎.vscode/settings.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@
55
"azureFunctions.projectRuntime": "~4",
66
"debug.internalConsoleOptions": "neverOpen",
77
"azureFunctions.projectSubpath": "app/indexer/functions",
8-
"azureFunctions.preDeployTask": "package (functions)"
8+
"azureFunctions.preDeployTask": "package (functions)",
9+
"java.configuration.updateBuildConfiguration": "interactive"
910
}

‎app/backend/Dockerfile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,7 @@ COPY --from=build ${DEPENDENCY}/BOOT-INF/lib /app/lib
1919
COPY --from=build ${DEPENDENCY}/META-INF /app/META-INF
2020
COPY --from=build ${DEPENDENCY}/BOOT-INF/classes /app
2121

22-
ENTRYPOINT ["java","-noverify", "-XX:MaxRAMPercentage=70", "-XX:+UseParallelGC", "-XX:ActiveProcessorCount=2", "-cp","app:app/lib/*","com.microsoft.openai.samples.rag.Application"]
22+
RUN curl -LJ -o /app/applicationinsights-agent-3.4.19.jar https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.4.19/applicationinsights-agent-3.4.19.jar
23+
COPY applicationinsights.json /app
24+
25+
ENTRYPOINT ["java","-javaagent:/app/applicationinsights-agent-3.4.19.jar","-noverify", "-XX:MaxRAMPercentage=70", "-XX:+UseParallelGC", "-XX:ActiveProcessorCount=2", "-cp","app:app/lib/*","com.microsoft.openai.samples.rag.Application"]

‎app/backend/applicationinsights.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"role": {
3+
"name": "api"
4+
}
5+
}

‎app/indexer/.mvn/jvm.config

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
2+
--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
3+
--add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
4+
--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED

‎app/indexer/.mvn/wrapper/maven-wrapper.jar

49.5 KB
Binary file not shown.

‎app/indexer/.mvn/wrapper/maven-wrapper.properties

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
2+
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.2/maven-wrapper-0.5.2.tar.gz

‎app/indexer/cli/src/main/java/com/microsoft/openai/samples/indexer/AddCommand.java

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.microsoft.openai.samples.indexer;
2+
3+
import com.microsoft.openai.samples.indexer.index.SearchIndexManager;
4+
import com.microsoft.openai.samples.indexer.storage.BlobManager;
5+
import org.slf4j.Logger;
6+
import org.slf4j.LoggerFactory;
7+
8+
import java.nio.file.Files;
9+
import java.nio.file.Path;
10+
11+
public class AddCommand {
12+
13+
private static final Logger logger = LoggerFactory.getLogger(AddCommand.class);
14+
private final SearchIndexManager searchIndexManager;
15+
private final BlobManager blobManager;
16+
private final DocumentProcessor documentProcessor;
17+
18+
public AddCommand(SearchIndexManager searchIndexManager, BlobManager blobManager, DocumentProcessor documentProcessor) {
19+
this.searchIndexManager = searchIndexManager;
20+
this.blobManager = blobManager;
21+
this.documentProcessor = documentProcessor;
22+
}
23+
24+
public void run(Path path,String category){
25+
26+
searchIndexManager.createIndex();
27+
28+
if(Files.isDirectory(path))
29+
processDirectory(path,category);
30+
else
31+
processFile(path,category);
32+
}
33+
34+
private void processDirectory(Path directory, String category) {
35+
logger.debug("Processing directory {}", directory);
36+
try {
37+
Files.newDirectoryStream(directory).forEach(path -> {
38+
processFile(path,category);
39+
});
40+
logger.debug("All files in directory {} processed", directory.toRealPath().toString());
41+
} catch (Exception e) {
42+
throw new RuntimeException("Error processing folder ",e);
43+
}
44+
}
45+
46+
private void processFile(Path path,String category) {
47+
try {
48+
String absoluteFilePath = path.toRealPath().toString();
49+
documentProcessor.indexDocumentfromFile(absoluteFilePath,category);
50+
logger.debug("file {} indexed", absoluteFilePath);
51+
blobManager.uploadBlob(path.toFile());
52+
logger.debug("file {} uploaded", absoluteFilePath);
53+
} catch (Exception e) {
54+
throw new RuntimeException("Error processing file ",e);
55+
}
56+
}
57+
}

‎app/indexer/cli/src/main/java/com/microsoft/openai/samples/indexer/CLI.java

Lines changed: 32 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public class CLI implements Callable<Integer> {
3838
@Option(names = {"--searchservice"}, required = true)
3939
private String searchservice;
4040

41-
@Option(names = {"--searchanalyzername"}, required = false, defaultValue = " en.microsoft")
41+
@Option(names = {"--searchanalyzername"}, required = false, defaultValue = "en.microsoft")
4242
private String searchanalyzername;
4343

4444
@Option(names = {"--index"}, required = true)
@@ -69,55 +69,48 @@ public static void main(String[] args) {
6969

7070
@Override
7171
public Integer call() throws Exception {
72-
System.out.println(" use add command");
72+
System.out.println(" use add or upload commands");
7373
return 0;
7474
}
7575

7676
@Command(name = "add")
7777
public void addCommand() {
7878
TokenCredential tokenCredential = new AzureDeveloperCliCredentialBuilder().build();
79+
SearchIndexManager searchIndexManager = buildSearchIndexManager(tokenCredential);
7980

80-
SearchIndexManager searchIndexManager = new SearchIndexManager(
81-
new AzureSearchClientFactory(searchservice, tokenCredential, index, verbose),
82-
searchanalyzername,
83-
new AzureOpenAIEmbeddingService(openaiServiceName, openaiEmbdeployment, tokenCredential, verbose));
84-
85-
searchIndexManager.createIndex();
86-
87-
//DocumentProcessor documentProcessor = new DocumentProcessor(searchIndexManager, new ItextPDFParser(), new TextSplitter(verbose));
88-
DocumentProcessor documentProcessor = new DocumentProcessor(searchIndexManager, new DocumentIntelligencePDFParser(formrecognizerservice,tokenCredential,verbose), new TextSplitter(verbose));
89-
BlobManager blobManager = new BlobManager(storageaccount, container, tokenCredential, verbose);
90-
91-
if(Files.isDirectory(dataFolderPath))
92-
processDirectory(documentProcessor, blobManager, dataFolderPath);
93-
else
94-
processFile(documentProcessor, blobManager, dataFolderPath);
95-
81+
new AddCommand(searchIndexManager,
82+
buildBlobManager(tokenCredential),
83+
buildDocumentProcessor(searchIndexManager, tokenCredential))
84+
.run(dataFolderPath,category);
9685
}
9786

98-
private void processDirectory(DocumentProcessor documentProcessor, BlobManager blobManager, Path directory) {
99-
logger.debug("Processing directory {}", directory);
100-
try {
101-
Files.newDirectoryStream(directory).forEach(path -> {
102-
processFile(documentProcessor, blobManager, path);
103-
});
104-
logger.debug("All files in directory {} processed", directory.toRealPath().toString());
105-
} catch (Exception e) {
106-
throw new RuntimeException("Error processing folder ",e);
87+
@Command(name = "upload")
88+
public void uploadCommand() {
89+
TokenCredential tokenCredential = new AzureDeveloperCliCredentialBuilder().build();
90+
91+
new UploadCommand(buildSearchIndexManager(tokenCredential),
92+
buildBlobManager(tokenCredential))
93+
.run(dataFolderPath,category);
94+
10795
}
108-
}
10996

110-
private void processFile(DocumentProcessor documentProcessor, BlobManager blobManager, Path path) {
111-
try {
112-
String absoluteFilePath = path.toRealPath().toString();
113-
documentProcessor.indexDocumentfromFile(absoluteFilePath,category);
114-
logger.debug("file {} indexed", absoluteFilePath);
115-
blobManager.uploadBlob(path.toFile());
116-
logger.debug("file {} uploaded", absoluteFilePath);
117-
} catch (Exception e) {
118-
throw new RuntimeException("Error processing file ",e);
119-
}
120-
}
97+
private BlobManager buildBlobManager(TokenCredential tokenCredential) {
98+
BlobManager blobManager = new BlobManager(storageaccount, container, tokenCredential, verbose);
99+
return blobManager;
100+
}
101+
102+
private DocumentProcessor buildDocumentProcessor(SearchIndexManager searchIndexManager, TokenCredential tokenCredential) {
103+
DocumentProcessor documentProcessor = new DocumentProcessor(searchIndexManager, new DocumentIntelligencePDFParser(formrecognizerservice, tokenCredential,verbose), new TextSplitter(verbose));
104+
return documentProcessor;
105+
}
106+
107+
private SearchIndexManager buildSearchIndexManager(TokenCredential tokenCredential) {
108+
SearchIndexManager searchIndexManager = new SearchIndexManager(
109+
new AzureSearchClientFactory(searchservice, tokenCredential, index, verbose),
110+
searchanalyzername,
111+
new AzureOpenAIEmbeddingService(openaiServiceName, openaiEmbdeployment, tokenCredential, verbose));
112+
return searchIndexManager;
113+
}
121114

122115

123116
}

‎app/indexer/cli/src/main/java/com/microsoft/openai/samples/indexer/UploadCommand.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.microsoft.openai.samples.indexer;
2+
3+
import com.microsoft.openai.samples.indexer.index.SearchIndexManager;
4+
import com.microsoft.openai.samples.indexer.storage.BlobManager;
5+
import org.slf4j.Logger;
6+
import org.slf4j.LoggerFactory;
7+
8+
import java.nio.file.Files;
9+
import java.nio.file.Path;
10+
11+
public class UploadCommand {
12+
13+
private static final Logger logger = LoggerFactory.getLogger(UploadCommand.class);
14+
private final SearchIndexManager searchIndexManager;
15+
private final BlobManager blobManager;
16+
17+
public UploadCommand(SearchIndexManager searchIndexManager, BlobManager blobManager) {
18+
this.searchIndexManager = searchIndexManager;
19+
this.blobManager = blobManager;
20+
}
21+
22+
public void run(Path path,String category){
23+
24+
searchIndexManager.createIndex();
25+
26+
if(Files.isDirectory(path))
27+
uploadDirectory(path);
28+
else
29+
uploadFile(path);
30+
}
31+
32+
private void uploadDirectory( Path directory) {
33+
logger.debug("Uploading directory {}", directory);
34+
try {
35+
Files.newDirectoryStream(directory).forEach(path -> {
36+
uploadFile(path);
37+
});
38+
logger.debug("All files in directory {} have been uploaded", directory.toRealPath().toString());
39+
} catch (Exception e) {
40+
throw new RuntimeException("Error processing folder ",e);
41+
}
42+
}
43+
44+
private void uploadFile(Path path) {
45+
try {
46+
String absoluteFilePath = path.toRealPath().toString();
47+
blobManager.uploadBlob(path.toFile());
48+
logger.debug("file {} uploaded", absoluteFilePath);
49+
} catch (Exception e) {
50+
throw new RuntimeException("Error processing file ",e);
51+
}
52+
}
53+
}

‎app/indexer/core/src/main/java/com/microsoft/openai/samples/indexer/embeddings/AbstractTextEmbeddingsService.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ public List<List<Double>> createEmbeddingBatch(List<String> texts) {
7070
for (EmbeddingItem data : embResponse.getData()) {
7171
embeddings.add(data.getEmbedding());
7272
}
73-
if (verbose) {
74-
logger.info("Embedding batch[%d] of [%d] completed. Batch size [%d] Token count [%d]".formatted(batchIndex+1, batches.size(), batch.getTexts().size(), batch.getTokenLength()));
75-
}
73+
74+
logger.info("Embedding batch[%d] of [%d] completed. Batch size [%d] Token count [%d]".formatted(batchIndex+1, batches.size(), batch.getTexts().size(), batch.getTokenLength()));
75+
7676

7777
}
7878

0 commit comments

Comments
 (0)