setup data dir

Author: jordanstephens

README.md

@@ -41,7 +41,7 @@ This application demonstrates how to deploy a Flask-based Retrieval-Augmented Ge
 ## Configuration
 
 - The knowledge base is the all the markdown files in the Defang docs [website](https://docs.defang.io/docs/intro). The logic for parsing can be found in `./app/get_knowledge_base.py`.
-- The file `get_knowledge_base.py` parses every webpage as specified into paragraphs and writes to `knowledge_base.json` for the RAG retrieval.
+- The file `get_knowledge_base.py` parses every webpage as specified into paragraphs and writes to `./data/knowledge_base.json` for the RAG retrieval.
 - To obtain your own knowledge base, please feel free to implement your own parsing scheme.
 - for local development, please use the `compose.dev.yaml` file where as for production, please use the `compose.yaml`.
 

app/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, request, jsonify, render_template, Response, stream_with_context, session
+from flask import Flask, request, jsonify, render_template, Response, stream_with_context, session, send_from_directory
 from flask_wtf.csrf import CSRFProtect
 from rag_system import rag_system
 import hashlib
@@ -116,6 +116,12 @@ def trigger_rebuild():
         print(f"Error in /trigger-rebuild endpoint: {e}")
         return jsonify({"error": "Internal Server Error"}), 500
 
+@app.route("/data/<path:name>")
+def download_file(name):
+    return send_from_directory(
+        "data", name, as_attachment=True
+    )
+
 if os.getenv('DEBUG') == '1':
     @app.route('/ask/debug', methods=['POST'])
     def debug_context():

app/knowledge_base.json renamed to app/data/knowledge_base.json

@@ -5,6 +5,8 @@
 import json
 from git import Repo
 
+kb_file_path = './data/knowledge_base.json'
+
 def clean_tmp(dir_path):
     """ Clears out all contents of the specified directory except for prebuild.sh """
     for item in os.listdir(dir_path):
@@ -80,14 +82,14 @@ def parse_markdown():
 
 def reset_knowledge_base():
     """ Resets or initializes the knowledge base JSON file. """
-    with open('./knowledge_base.json', 'w') as output_file:
+    with open(kb_file_path, 'w') as output_file:
         json.dump([], output_file)
 
 def parse_markdown_file_to_json(file_path):
     """ Parses individual markdown file and adds its content to JSON """
     try:
         # Load existing content if the file exists
-        with open('./knowledge_base.json', 'r') as existing_file:
+        with open(kb_file_path, 'r') as existing_file:
             json_output = json.load(existing_file)
             current_id = len(json_output) + 1  # Start ID from the next available number
     except (FileNotFoundError, json.JSONDecodeError):
@@ -147,15 +149,15 @@ def parse_markdown_file_to_json(file_path):
             })
             current_id += 1
 
-    # Write the augmented JSON output to knowledge_base.json
-    with open('./knowledge_base.json', 'w', encoding='utf-8') as output_file:
+    # Write the augmented JSON output to ./data/knowledge_base.json
+    with open(kb_file_path, 'w', encoding='utf-8') as output_file:
         json.dump(json_output, output_file, indent=2, ensure_ascii=False)
 
 def parse_cli_markdown(file_path):
     """ Parses CLI-specific markdown files """
     try:
         # Load existing content if the file exists
-        with open('./knowledge_base.json', 'r') as existing_file:
+        with open(kb_file_path, 'r') as existing_file:
             json_output = json.load(existing_file)
             current_id = len(json_output) + 1  # Start ID from the next available number
     except (FileNotFoundError, json.JSONDecodeError):
@@ -187,8 +189,8 @@ def parse_cli_markdown(file_path):
         })
         current_id += 1
 
-    # Write the augmented JSON output to knowledge_base.json
-    with open('./knowledge_base.json', 'w', encoding='utf-8') as output_file:
+    # Write the augmented JSON output to data/knowledge_base.json
+    with open(kb_file_path, 'w', encoding='utf-8') as output_file:
         json.dump(json_output, output_file, indent=2, ensure_ascii=False)
 
 def recursive_parse_directory(root_dir):

app/get_knowledge_base.py

@@ -12,7 +12,7 @@
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
 class RAGSystem:
-    def __init__(self, knowledge_base_path='knowledge_base.json'):
+    def __init__(self, knowledge_base_path='./data/knowledge_base.json'):
         self.knowledge_base_path = knowledge_base_path
         self.knowledge_base = self.load_knowledge_base()
         self.model = SentenceTransformer('all-MiniLM-L6-v2')