Refactor scripts to avoid anti-patterns, redundancy (#1986)

* Update bicep for ACA

* First working version

* Support workload profile

* Add support for CORS and fix identity for openai

* Add aca-host

* Make acr unique

* Add doc for aca host

* Update ACA docs

* Remove unneeded bicep files

* Revert chanes to infra/main.parameters.json

* Fix markdown lint issues

* Run frontend build before building docker image

* remove symlinks and update scripts with paths relative to its own folder instead of cwd

* Merge with main.bicep

* output AZURE_CONTAINER_REGISTRY_ENDPOINT

* Fix deployment with app service

* Improve naming and README

* Fix identity name and cost esitmation for aca

* Share env vars in bicep and update docs

* Revert "remove symlinks and update scripts with paths relative to its own folder instead of cwd"

This reverts commit 40287f2.

* Add containerapps as a commented out host option

* Update app/backend/.dockerignore

* Apply suggestions from code review

* More steps for deployment guide

* Update azure.yaml

* Update comment

* cleanup bicep files and improve docs

* Update condition for running in production for credential

* Refactors to scripts

* Remove phi changes

* Make mypy happy

* Add dotenv requirement

* Env var tweaks

* Fix error handling

* Update manageacl.py commands

* Doc update

* Adding more tests for prepdocs

* Fix markdown copy

* Fix relative links

* Make prepdocs mypy happy

* Fix auth_update if check

---------

Co-authored-by: yefuwang <[email protected]>
Co-authored-by: Yefu Wang <[email protected]>

Author: pamelafox

app/backend/app.py

@@ -714,9 +714,10 @@ def create_app():
     # Log levels should be one of https://docs.python.org/3/library/logging.html#logging-levels
     # Set root level to WARNING to avoid seeing overly verbose logs from SDKS
     logging.basicConfig(level=logging.WARNING)
-    # Set the app logger level to INFO by default
-    default_level = "INFO"
-    app.logger.setLevel(os.getenv("APP_LOG_LEVEL", default_level))
+    # Set our own logger levels to INFO by default
+    app_level = os.getenv("APP_LOG_LEVEL", "INFO")
+    app.logger.setLevel(os.getenv("APP_LOG_LEVEL", app_level))
+    logging.getLogger("scripts").setLevel(app_level)
 
     if allowed_origin := os.getenv("ALLOWED_ORIGIN"):
         app.logger.info("ALLOWED_ORIGIN is set, enabling CORS for %s", allowed_origin)

app/backend/load_azd_env.py

@@ -0,0 +1,23 @@
+import json
+import logging
+import subprocess
+
+from dotenv import load_dotenv
+
+logger = logging.getLogger("scripts")
+
+
+def load_azd_env():
+    """Get path to current azd env file and load file using python-dotenv"""
+    result = subprocess.run("azd env list -o json", shell=True, capture_output=True, text=True)
+    if result.returncode != 0:
+        raise Exception("Error loading azd env")
+    env_json = json.loads(result.stdout)
+    env_file_path = None
+    for entry in env_json:
+        if entry["IsDefault"]:
+            env_file_path = entry["DotEnvPath"]
+    if not env_file_path:
+        raise Exception("No default azd env file found")
+    logger.info(f"Loading azd env from {env_file_path}")
+    load_dotenv(env_file_path, override=True)

app/backend/main.py

@@ -1,3 +1,12 @@
+import os
+
 from app import create_app
+from load_azd_env import load_azd_env
+
+# WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep
+RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None
+
+if not RUNNING_ON_AZURE:
+    load_azd_env()
 
 app = create_app()

app/backend/prepdocs.py

@@ -18,7 +18,7 @@
 
 from .listfilestrategy import File
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class BlobManager:

app/backend/prepdocslib/blobmanager.py

@@ -17,7 +17,7 @@
 )
 from typing_extensions import TypedDict
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class EmbeddingBatch:

app/backend/prepdocslib/embeddings.py

@@ -8,7 +8,7 @@
 from .searchmanager import SearchManager, Section
 from .strategy import DocumentAction, SearchInfo, Strategy
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 async def parse_file(

app/backend/prepdocslib/filestrategy.py

@@ -7,7 +7,7 @@
 from .page import Page
 from .parser import Parser
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 def cleanup_data(data: str) -> str:

app/backend/prepdocslib/htmlparser.py

@@ -28,7 +28,7 @@
 from .searchmanager import SearchManager
 from .strategy import DocumentAction, SearchInfo, Strategy
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class IntegratedVectorizerStrategy(Strategy):

app/backend/prepdocslib/integratedvectorizerstrategy.py

@@ -13,7 +13,7 @@
     DataLakeServiceClient,
 )
 
-logger = logging.getLogger("ingester")
+logger = logging.getLogger("scripts")
 
 
 class File: