Allow the setting of log level by env vars (#642)

* Allow the setting of log level by env vars

Required by #637

* Allow the setting of log level by env vars

Required by #637

* Add loglevel to env.sample

* Allow log level to be set via bicep

* Set log level in admin app

* Correctly configure logs in web app

* Ensure log level is set for functions

* Remove duplicated tests

* Fix main.json after merge

* Reduce verbose azure logs

Author: adamdougal

.env.sample

@@ -55,3 +55,4 @@ AZURE_SPEECH_SERVICE_REGION=
 AZURE_AUTH_TYPE=keys
 USE_KEY_VAULT=true
 AZURE_KEY_VAULT_ENDPOINT=
+LOGLEVEL=INFO

code/app.py

@@ -1,7 +1,11 @@
 import os
+import logging
 from azure.monitor.opentelemetry import configure_azure_monitor
 from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
 
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO").upper())
+# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
+logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
 # We cannot use EnvHelper here as Application Insights should be configured first
 # for instrumentation to work correctly
 if os.getenv("APPINSIGHTS_ENABLED", "false").lower() == "true":

code/backend/Admin.py

@@ -10,14 +10,16 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
+logging.basicConfig(level=os.getenv("LOGLEVEL", "INFO").upper())
+# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
+logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
 # We cannot use EnvHelper here as Application Insights needs to be configured first
 # for instrumentation to work correctly
 if os.getenv("APPINSIGHTS_ENABLED", "false").lower() == "true":
     configure_azure_monitor()
 
-logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
-    logging.WARNING
-)
+logger = logging.getLogger(__name__)
+logger.debug("Starting admin app")
 
 
 st.set_page_config(

code/backend/batch/AddURLEmbeddings.py

@@ -3,14 +3,17 @@
 import azure.functions as func
 import sys
 
+from utilities.helpers.EnvHelper import EnvHelper
 from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
 from utilities.helpers.ConfigHelper import ConfigHelper
 
 sys.path.append("..")
 
 bp_add_url_embeddings = func.Blueprint()
+env_helper: EnvHelper = EnvHelper()
 
 logger = logging.getLogger(__name__)
+logger.setLevel(env_helper.LOGLEVEL)
 
 
 @bp_add_url_embeddings.route(route="AddURLEmbeddings")

code/backend/batch/BatchPushResults.py

@@ -4,15 +4,18 @@
 from urllib.parse import urlparse
 import sys
 
+from utilities.helpers.EnvHelper import EnvHelper
 from utilities.helpers.AzureBlobStorageHelper import AzureBlobStorageClient
 from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
 from utilities.helpers.ConfigHelper import ConfigHelper
 
 sys.path.append("..")
 
 bp_batch_push_results = func.Blueprint()
+env_helper: EnvHelper = EnvHelper()
 
 logger = logging.getLogger(__name__)
+logger.setLevel(env_helper.LOGLEVEL)
 
 
 def _get_file_name_from_message(msg: func.QueueMessage) -> str:

code/backend/batch/BatchStartProcessing.py

@@ -13,6 +13,7 @@
 env_helper: EnvHelper = EnvHelper()
 
 logger = logging.getLogger(__name__)
+logger.setLevel(env_helper.LOGLEVEL)
 
 
 @bp_batch_start_processing.route(route="BatchStartProcessing")

code/backend/batch/GetConversationResponse.py

@@ -11,6 +11,7 @@
 env_helper: EnvHelper = EnvHelper()
 
 logger = logging.getLogger(__name__)
+logger.setLevel(env_helper.LOGLEVEL)
 
 
 @bp_get_conversation_response.route(route="GetConversationResponse")

code/backend/batch/function_app.py

@@ -1,10 +1,14 @@
+import logging
+import os
 import azure.functions as func
 from AddURLEmbeddings import bp_add_url_embeddings
 from BatchPushResults import bp_batch_push_results
 from BatchStartProcessing import bp_batch_start_processing
 from GetConversationResponse import bp_get_conversation_response
 from azure.monitor.opentelemetry import configure_azure_monitor
 
+# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
+logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
 configure_azure_monitor()
 
 app = func.FunctionApp(

code/backend/batch/utilities/helpers/EnvHelper.py

@@ -14,6 +14,8 @@ def __init__(self, **kwargs) -> None:
         # Wrapper for Azure Key Vault
         self.secretHelper = SecretHelper()
 
+        self.LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
+
         # Azure Search
         self.AZURE_SEARCH_SERVICE = os.getenv("AZURE_SEARCH_SERVICE", "")
         self.AZURE_SEARCH_INDEX = os.getenv("AZURE_SEARCH_INDEX", "")

code/backend/pages/01_Ingest_Data.py

@@ -5,7 +5,6 @@
 import traceback
 import chardet
 from datetime import datetime, timedelta
-import logging
 import requests
 from azure.identity import DefaultAzureCredential
 from azure.storage.blob import (
@@ -22,9 +21,6 @@
 sys.path.append(path.join(path.dirname(__file__), ".."))
 env_helper: EnvHelper = EnvHelper()
 
-logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
-    logging.WARNING
-)
 st.set_page_config(
     page_title="Ingest Data",
     page_icon=path.join("images", "favicon.ico"),