From 7af05c8e454a9e7934013be3be8186b0899a6db1 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Mon, 31 Mar 2025 03:08:14 +0000 Subject: [PATCH 01/30] make deployable --- infra/core/storage/storage-account.bicep | 2 +- infra/main.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/core/storage/storage-account.bicep b/infra/core/storage/storage-account.bicep index 6149fb2..7709e21 100644 --- a/infra/core/storage/storage-account.bicep +++ b/infra/core/storage/storage-account.bicep @@ -10,7 +10,7 @@ param tags object = {} param accessTier string = 'Hot' param allowBlobPublicAccess bool = true param allowCrossTenantReplication bool = true -param allowSharedKeyAccess bool = true +param allowSharedKeyAccess bool = false param containers array = [] param corsRules array = [] param defaultToOAuthAuthentication bool = false diff --git a/infra/main.bicep b/infra/main.bicep index bdd8dab..19a8a3d 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -119,7 +119,7 @@ param seed string = newGuid() var abbrs = loadJsonContent('./abbreviations.json') var resourceToken = toLower(uniqueString(subscription().id, environmentName, location, seed)) var projectName = !empty(aiProjectName) ? aiProjectName : 'ai-project-${resourceToken}' -var tags = { 'azd-env-name': environmentName } +var tags = { 'azd-env-name': environmentName, 'OwningExPTrack': '3P' } var agentID = !empty(aiAgentID) ? aiAgentID : '' From b399a5103e7cc225d10a19f7bb3cfbbf25de8d7a Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 03:29:17 +0000 Subject: [PATCH 02/30] minimal updates --- .vscode/launch.json | 34 ++++++++++++++++++++++++++++++++++ src/Dockerfile | 4 ++-- src/api/main.py | 2 +- 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..dbffc2a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,34 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Python: FastAPI", + "type": "debugpy", + "request": "launch", + "module": "uvicorn", + "cwd": "${workspaceFolder}/src", + "args": [ + "api.main:create_app", + "--host", + "127.0.0.1", + "--port", + "8000", + "--factory", + "--reload" + ], + "justMyCode": false + }, + { + "name": "Python Debugger: Current File", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/src/Dockerfile b/src/Dockerfile index ea79389..fc5cec4 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -6,9 +6,9 @@ WORKDIR /code COPY . . -ENV ENABLE_AZURE_MONITOR_TRACING=false +ENV ENABLE_AZURE_MONITOR_TRACING=true -ENV AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED=false +ENV AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED=true #ENV APP_LOG_FILE=app.log diff --git a/src/api/main.py b/src/api/main.py index 02157e6..fe0734c 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -49,7 +49,7 @@ async def lifespan(app: fastapi.FastAPI): from azure.monitor.opentelemetry import configure_azure_monitor configure_azure_monitor(connection_string=application_insights_connection_string) # Do not instrument the code yet, before trace fix is available. - #ai_client.telemetry.enable() + ai_client.telemetry.enable() if os.environ.get("AZURE_AI_AGENT_ID"): try: From bb08f51fd4234de605e2f6f5d3cdbf9e4c333253 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 03:35:30 +0000 Subject: [PATCH 03/30] remove vscode settings --- .vscode/launch.json | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index dbffc2a..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - { - "name": "Python: FastAPI", - "type": "debugpy", - "request": "launch", - "module": "uvicorn", - "cwd": "${workspaceFolder}/src", - "args": [ - "api.main:create_app", - "--host", - "127.0.0.1", - "--port", - "8000", - "--factory", - "--reload" - ], - "justMyCode": false - }, - { - "name": "Python Debugger: Current File", - "type": "debugpy", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": false - } - ] -} \ No newline at end of file From 371b4c326110d97fa8abae0939361bdfc498141a Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 03:36:13 +0000 Subject: [PATCH 04/30] add to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 63257df..9757137 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ ENV/ env.bak/ venv.bak/ .azure +.vscode/ \ No newline at end of file From c2b3b63b8f527ac6725df109cb145f1918a8e31e Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 08:46:22 +0000 Subject: [PATCH 05/30] Work in progress --- .config/feature-flags.json.sample | 40 +++++++++++++++++++++++ .github/workflows/experiment-deploy.yml | 40 +++++++++++++++++++++++ .github/workflows/experiment-validate.yml | 31 ++++++++++++++++++ .github/workflows/template-validation.yml | 1 + azure.yaml | 3 +- infra/core/config/configstore.bicep | 24 +++++++++++++- infra/main.bicep | 30 +++++++++++++++-- scripts/write_env.ps1 | 2 ++ scripts/write_env.sh | 1 + src/.env.sample | 2 ++ src/api/main.py | 25 +++++++++++++- src/api/routes.py | 27 +++++++++++++-- src/requirements.txt | 6 ++++ 13 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 .config/feature-flags.json.sample create mode 100644 .github/workflows/experiment-deploy.yml create mode 100644 .github/workflows/experiment-validate.yml diff --git a/.config/feature-flags.json.sample b/.config/feature-flags.json.sample new file mode 100644 index 0000000..eff283d --- /dev/null +++ b/.config/feature-flags.json.sample @@ -0,0 +1,40 @@ +{ + "schemaVersion": "2.0.0", + "feature_management": { + "feature_flags": [ + { + "id": "my-agent", + "enabled": true, + "variants": [ + { + "name": "v1", + "configuration_value": "" + }, + { + "name": "v2", + "configuration_value": "" + } + ], + "allocation": { + "percentile": [ + { + "variant": "v1", + "from": 0, + "to": 50 + }, + { + "variant": "v2", + "from": 50, + "to": 100 + } + ], + "default_when_enabled": "v1", + "default_when_disabled": "v1" + }, + "telemetry": { + "enabled": true + } + } + ] + } +} \ No newline at end of file diff --git a/.github/workflows/experiment-deploy.yml b/.github/workflows/experiment-deploy.yml new file mode 100644 index 0000000..d829c12 --- /dev/null +++ b/.github/workflows/experiment-deploy.yml @@ -0,0 +1,40 @@ +name: Deploy Experiments +on: + workflow_dispatch: + push: + +# GitHub Actions workflow to deploy to Azure using azd +# To configure required secrets for connecting to Azure, simply run `azd pipeline config` + +# Set up permissions for deploying with secretless Azure federated credentials +# https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication +permissions: + id-token: write + contents: read + +env: + APP_CONFIGURATION_FILE: .config/feature-flags*.json + +jobs: + deploy-experiments: + name: Deploy Experiments + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Azure login using Federated Credentials + uses: azure/login@v2 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + enable-AzPSSession: true + + - name: Deploy App Config feature flags + uses: azure/app-configuration-deploy-feature-flags@v1-beta + with: + path: ${{ env.APP_CONFIGURATION_FILE }} + app-configuration-endpoint: ${{ vars.APP_CONFIGURATION_ENDPOINT }} + strict: false + \ No newline at end of file diff --git a/.github/workflows/experiment-validate.yml b/.github/workflows/experiment-validate.yml new file mode 100644 index 0000000..4cd8c6d --- /dev/null +++ b/.github/workflows/experiment-validate.yml @@ -0,0 +1,31 @@ +name: Deploy Experiments +on: + workflow_dispatch: + push: + +# GitHub Actions workflow to deploy to Azure using azd +# To configure required secrets for connecting to Azure, simply run `azd pipeline config` + +# Set up permissions for deploying with secretless Azure federated credentials +# https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication +permissions: + id-token: write + contents: read + +env: + APP_CONFIGURATION_FILE: .config/feature-flags*.json + +jobs: + validate-feature-flags: + name: Validate Feature Flags + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate App Config feature flags + uses: azure/app-configuration-deploy-feature-flags@v1-beta + with: + path: ${{ env.APP_CONFIGURATION_FILE }} + operation: validate + \ No newline at end of file diff --git a/.github/workflows/template-validation.yml b/.github/workflows/template-validation.yml index 9b05445..39e2a46 100644 --- a/.github/workflows/template-validation.yml +++ b/.github/workflows/template-validation.yml @@ -52,5 +52,6 @@ jobs: AZURE_AI_EMBED_MODEL_FORMAT: ${{ vars.AZURE_AI_EMBED_MODEL_FORMAT }} AZURE_AI_EMBED_MODEL_VERSION: ${{ vars.AZURE_AI_EMBED_MODEL_VERSION }} AZURE_EXISTING_AIPROJECT_CONNECTION_STRING: ${{ vars.AZURE_EXISTING_AIPROJECT_CONNECTION_STRING }} + APP_CONFIGURATION_ENDPOINT: ${{ vars.APP_CONFIGURATION_ENDPOINT }} - name: print result run: cat ${{ steps.validation.outputs.resultFile }} diff --git a/azure.yaml b/azure.yaml index 91e8ebc..8e82ee6 100644 --- a/azure.yaml +++ b/azure.yaml @@ -50,4 +50,5 @@ pipeline: - AZURE_AI_EMBED_MODEL_VERSION - AZURE_AI_EMBED_DIMENSIONS - AZURE_AI_SEARCH_INDEX_NAME - - AZURE_EXISTING_AIPROJECT_CONNECTION_STRING \ No newline at end of file + - AZURE_EXISTING_AIPROJECT_CONNECTION_STRING + - APP_CONFIGURATION_ENDPOINT \ No newline at end of file diff --git a/infra/core/config/configstore.bicep b/infra/core/config/configstore.bicep index 96818f1..e5cbfbe 100644 --- a/infra/core/config/configstore.bicep +++ b/infra/core/config/configstore.bicep @@ -18,13 +18,30 @@ param keyValueValues array = [] @description('The principal ID to grant access to the Azure App Configuration store') param principalId string -resource configStore 'Microsoft.AppConfiguration/configurationStores@2023-03-01' = { +@description('The Application Insights ID linked to the Azure App Configuration store') +param appInsightsName string + +// TODO: enable experimentation +resource configStore 'Microsoft.AppConfiguration/configurationStores@2023-09-01-preview' = { name: name location: location sku: { name: 'standard' } tags: tags + properties: { + encryption: {} + disableLocalAuth: true + enablePurgeProtection: false + experimentation:{} + dataPlaneProxy:{ + authenticationMode: 'Pass-through' + privateLinkDelegation: 'Disabled' + } + telemetry: { + resourceId: appInsights.id + } + } } resource configStoreKeyValue 'Microsoft.AppConfiguration/configurationStores/keyValues@2023-03-01' = [for (item, i) in keyValueNames: { @@ -45,4 +62,9 @@ module configStoreAccess '../security/configstore-access.bicep' = { dependsOn: [configStore] } +resource appInsights 'Microsoft.Insights/components@2020-02-02-preview' existing = { + name: appInsightsName +} + output endpoint string = configStore.properties.endpoint +output name string = configStore.name diff --git a/infra/main.bicep b/infra/main.bicep index d901c24..00902a8 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -108,13 +108,16 @@ param useApplicationInsights bool = true @description('Do we want to use the Azure AI Search') param useSearchService bool = false +@description('Id of the user or app to assign application roles') +param principalId string = '' + @description('Random seed to be used during generation of new resources suffixes.') -param seed string = newGuid() +param seed string = '' //newGuid() why Guid? var abbrs = loadJsonContent('./abbreviations.json') var resourceToken = toLower(uniqueString(subscription().id, environmentName, location, seed)) var projectName = !empty(aiProjectName) ? aiProjectName : 'ai-project-${resourceToken}' -var tags = { 'azd-env-name': environmentName, 'OwningExPTrack': '3P' } +var tags = { 'azd-env-name': environmentName, 'OwningExPTrack': '3P' } // TODO: remove ExP tag var agentID = !empty(aiAgentID) ? aiAgentID : '' @@ -388,6 +391,28 @@ module backendRoleAzureAIDeveloperRG 'core/security/role.bicep' = { } } +// App Configuration +module configStore 'core/config/configstore.bicep' = { + name: 'config-store' + scope: rg + params: { + location: location + name: '${abbrs.appConfigurationStores}${resourceToken}' + tags: tags + principalId: api.outputs.SERVICE_API_IDENTITY_PRINCIPAL_ID + appInsightsName: ai.outputs.applicationInsightsName + } +} + +module configStoreDataOwnerAccess 'core/security/role.bicep' = { + scope: rg + name: 'config-store-data-owner-role' + params: { + principalId: principalId + roleDefinitionId: '5ae67dd6-50cb-40e7-96ff-dc2bfa4b606b' // App Configuration Data Owner + } +} + output AZURE_RESOURCE_GROUP string = rg.name // Outputs required for local development server @@ -401,6 +426,7 @@ output AZURE_AI_SEARCH_ENDPOINT string = searchServiceEndpoint output AZURE_AI_EMBED_DIMENSIONS string = embeddingDeploymentDimensions output AZURE_AI_AGENT_NAME string = agentName output AZURE_AI_AGENT_ID string = agentID +output APP_CONFIGURATION_ENDPOINT string = configStore.outputs.endpoint // Outputs required by azd for ACA output AZURE_CONTAINER_ENVIRONMENT_NAME string = containerApps.outputs.environmentName diff --git a/scripts/write_env.ps1 b/scripts/write_env.ps1 index aec2099..865a163 100644 --- a/scripts/write_env.ps1 +++ b/scripts/write_env.ps1 @@ -16,6 +16,7 @@ $azureAIEmbedDimensions = azd env get-value AZURE_AI_EMBED_DIMENSIONS $azureAISearchIndexName = azd env get-value AZURE_AI_SEARCH_INDEX_NAME $azureAISearchEndpoint = azd env get-value AZURE_AI_SEARCH_ENDPOINT $serviceAPIUri = azd env get-value SERVICE_API_URI +$appConfigurationEndpoint = azd env get-value APP_CONFIGURATION_ENDPOINT Add-Content -Path $envFilePath -Value "AZURE_AIPROJECT_CONNECTION_STRING=$azureAiProjectConnectionString" Add-Content -Path $envFilePath -Value "AZURE_AI_AGENT_DEPLOYMENT_NAME=$azureAiagentDeploymentName" @@ -28,6 +29,7 @@ Add-Content -Path $envFilePath -Value "AZURE_AI_SEARCH_INDEX_NAME=$azureAISearch Add-Content -Path $envFilePath -Value "AZURE_AI_SEARCH_ENDPOINT=$azureAISearchEndpoint" Add-Content -Path $envFilePath -Value "AZURE_AI_AGENT_NAME=$azureAiAgentName" Add-Content -Path $envFilePath -Value "AZURE_TENANT_ID=$azureTenantId" +Add-Content -Path $envFilePath -Value "APP_CONFIGURATION_ENDPOINT=$appConfigurationEndpoint" Write-Host "Web app URL:" Write-Host $serviceAPIUri -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/write_env.sh b/scripts/write_env.sh index fb298a6..615fe83 100755 --- a/scripts/write_env.sh +++ b/scripts/write_env.sh @@ -17,6 +17,7 @@ echo "AZURE_AI_SEARCH_INDEX_NAME=$(azd env get-value AZURE_AI_SEARCH_INDEX_NAME) echo "AZURE_AI_SEARCH_ENDPOINT=$(azd env get-value AZURE_AI_SEARCH_ENDPOINT)" >> $ENV_FILE_PATH echo "AZURE_AI_AGENT_NAME=$(azd env get-value AZURE_AI_AGENT_NAME)" >> $ENV_FILE_PATH echo "AZURE_TENANT_ID=$(azd env get-value AZURE_TENANT_ID)" >> $ENV_FILE_PATH +echo "APP_CONFIGURATION_ENDPOINT=$(azd env get-value APP_CONFIGURATION_ENDPOINT)" >> $ENV_FILE_PATH echo "Web app URL:" echo -e "\033[0;36m $(azd env get-value SERVICE_API_URI)" diff --git a/src/.env.sample b/src/.env.sample index 94ae896..59df28f 100644 --- a/src/.env.sample +++ b/src/.env.sample @@ -12,3 +12,5 @@ AZURE_AI_SEARCH_INDEX_NAME="" # required for index search. Example: "index_samp # Highly recommended. Example: "asst_AbCdEfGhIjKlMnOpQrStUvWxYz". If not specified, the agent name will be used to find the agent ID. Agent ID can be found by following https://learn.microsoft.com/en-us/azure/ai-services/agents/quickstart?pivots=ai-foundry-portal AZURE_AI_AGENT_ID="" +ENABLE_AZURE_MONITOR_TRACING=true +AZURE_TRACING_GEN_AI_CONTENT_RECORDING_ENABLED=true \ No newline at end of file diff --git a/src/api/main.py b/src/api/main.py index fe0734c..c9ffb9d 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -48,8 +48,31 @@ async def lifespan(app: fastapi.FastAPI): else: from azure.monitor.opentelemetry import configure_azure_monitor configure_azure_monitor(connection_string=application_insights_connection_string) - # Do not instrument the code yet, before trace fix is available. ai_client.telemetry.enable() + logger.info("Configured Application Insights for tracing.") + + app_config_conn_str = os.getenv("APP_CONFIGURATION_ENDPOINT") + if app_config_conn_str: + try: + from azure.appconfiguration.provider import load + from featuremanagement import FeatureManager + from featuremanagement.azuremonitor import publish_telemetry + app_config = load( + endpoint=app_config_conn_str, + credential=DefaultAzureCredential(), + feature_flag_enabled=True, + feature_flag_refresh_enabled=True, # Is it needed? + refresh_interval=60, # wait at least 60s before next refresh + ) + feature_manager = FeatureManager(app_config, on_feature_evaluated=publish_telemetry) + app.state.app_config = app_config + app.state.feature_manager = feature_manager + logger.info("Configured App Configuration with feature flag support.") + except ModuleNotFoundError: + logger.warning("Required libraries for App Configuration not installed.") + logger.warning("Please make sure azure-appconfiguration-provider and FeatureManagement are installed.") + except Exception as e: + logger.warning("Failed to setup App Configuration", exc_info=True) if os.environ.get("AZURE_AI_AGENT_ID"): try: diff --git a/src/api/routes.py b/src/api/routes.py index e22381e..8d430bd 100644 --- a/src/api/routes.py +++ b/src/api/routes.py @@ -44,10 +44,14 @@ def get_ai_client(request: Request) -> AIProjectClient: return request.app.state.ai_client - def get_agent(request: Request) -> Agent: return request.app.state.agent +def get_feature_manager(request: Request): + return getattr(request.app.state, "feature_manager", None) + +def get_app_config(request: Request): + return getattr(request.app.state, "app_config", None) def serialize_sse_event(data: Dict) -> str: return f"data: {json.dumps(data)}\n\n" @@ -135,6 +139,18 @@ async def index(request: Request): return templates.TemplateResponse("index.html", {"request": request}) +async def get_agent_variant(feature_manager, ai_client: AIProjectClient) -> str: + if feature_manager: + agent_variant = feature_manager.get_variant("my-agent") + if agent_variant and agent_variant.configuration: + try: + agent_variant = await ai_client.agents.get_agent(agent_variant.configuration) + logger.info(f"Using agent variant: {agent_variant.id}") + return agent_variant.id + except Exception as e: + logger.error(f"Error retrieving agent variant with Id {agent_variant.id}. {e}") + return None + async def get_result(thread_id: str, agent_id: str, ai_client : AIProjectClient) -> AsyncGenerator[str, None]: logger.info(f"get_result invoked for thread_id={thread_id} and agent_id={agent_id}") try: @@ -211,7 +227,13 @@ async def chat( request: Request, ai_client : AIProjectClient = Depends(get_ai_client), agent : Agent = Depends(get_agent), + feature_manager = Depends(get_feature_manager), + app_config = Depends(get_app_config), ): + # Refresh config if configured + #if app_config: + #app_config.refresh() + # Retrieve the thread ID from the cookies (if available). thread_id = request.cookies.get('thread_id') agent_id = request.cookies.get('agent_id') @@ -229,7 +251,8 @@ async def chat( raise HTTPException(status_code=400, detail=f"Error handling thread: {e}") thread_id = thread.id - agent_id = agent.id + #agent_id = await get_agent_variant(feature_manager, ai_client) or agent.id + agent_id = agent.id # Parse the JSON from the request. try: diff --git a/src/requirements.txt b/src/requirements.txt index 163f075..0084f0e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -8,3 +8,9 @@ azure-core-tracing-opentelemetry azure-monitor-opentelemetry azure-search-documents opentelemetry-sdk + +azure-ai-evaluation +azure-appconfiguration-provider==2.0.0b3 +FeatureManagement==2.0.0b3 +azure-monitor-opentelemetry-exporter +azure-monitor-events-extension \ No newline at end of file From 19abd4c46e0f4794b174c380f501fca7aad87000 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:38:32 +0000 Subject: [PATCH 06/30] Configure Azure Developer Pipeline From ba41c7ca935535ae620571b6938d12880c912bd3 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:44:04 +0000 Subject: [PATCH 07/30] add empty feature flag file --- .config/feature-flags.json | 7 +++++++ .github/workflows/experiment-deploy.yml | 2 +- .github/workflows/experiment-validate.yml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .config/feature-flags.json diff --git a/.config/feature-flags.json b/.config/feature-flags.json new file mode 100644 index 0000000..a6d5912 --- /dev/null +++ b/.config/feature-flags.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": "2.0.0", + "feature_management": { + "feature_flags": [ + ] + } +} \ No newline at end of file diff --git a/.github/workflows/experiment-deploy.yml b/.github/workflows/experiment-deploy.yml index d829c12..c83db81 100644 --- a/.github/workflows/experiment-deploy.yml +++ b/.github/workflows/experiment-deploy.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - APP_CONFIGURATION_FILE: .config/feature-flags*.json + APP_CONFIGURATION_FILE: .config/feature-flags.json jobs: deploy-experiments: diff --git a/.github/workflows/experiment-validate.yml b/.github/workflows/experiment-validate.yml index 4cd8c6d..2d4c0f2 100644 --- a/.github/workflows/experiment-validate.yml +++ b/.github/workflows/experiment-validate.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - APP_CONFIGURATION_FILE: .config/feature-flags*.json + APP_CONFIGURATION_FILE: .config/feature-flags.json jobs: validate-feature-flags: From c96b5f30dad12708b12191f8bb23d7e02477592f Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:45:31 +0000 Subject: [PATCH 08/30] Update workflow name --- .github/workflows/experiment-validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/experiment-validate.yml b/.github/workflows/experiment-validate.yml index 2d4c0f2..1a916c3 100644 --- a/.github/workflows/experiment-validate.yml +++ b/.github/workflows/experiment-validate.yml @@ -1,4 +1,4 @@ -name: Deploy Experiments +name: Validate Experiments on: workflow_dispatch: push: From 3fe9f12db79ad79c574cd00a9f8b61abc2f22172 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Thu, 10 Apr 2025 19:53:22 +0000 Subject: [PATCH 09/30] start a feature flag --- .config/feature-flags.json | 43 +++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/.config/feature-flags.json b/.config/feature-flags.json index a6d5912..0dd9a4e 100644 --- a/.config/feature-flags.json +++ b/.config/feature-flags.json @@ -1,7 +1,40 @@ { - "schemaVersion": "2.0.0", - "feature_management": { - "feature_flags": [ - ] - } + "schemaVersion": "2.0.0", + "feature_management": { + "feature_flags": [ + { + "id": "my-agent", + "enabled": true, + "variants": [ + { + "name": "v1", + "configuration_value": "asst_qLmkrF8NEhh3HDhky8nWRVDk" + }, + { + "name": "v2", + "configuration_value": "asst_MSuB3GboqgJNw74ogzHDD2cQ" + } + ], + "allocation": { + "percentile": [ + { + "variant": "v1", + "from": 0, + "to": 50 + }, + { + "variant": "v2", + "from": 50, + "to": 100 + } + ], + "default_when_enabled": "v1", + "default_when_disabled": "v1" + }, + "telemetry": { + "enabled": true + } + } + ] + } } \ No newline at end of file From e4e1de10035484478924f7c9827db870356ce319 Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Sun, 13 Apr 2025 22:32:02 +0000 Subject: [PATCH 10/30] minor updates --- src/api/main.py | 3 +-- src/api/routes.py | 9 ++++----- src/requirements.txt | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/api/main.py b/src/api/main.py index c9ffb9d..93ede71 100644 --- a/src/api/main.py +++ b/src/api/main.py @@ -61,8 +61,7 @@ async def lifespan(app: fastapi.FastAPI): endpoint=app_config_conn_str, credential=DefaultAzureCredential(), feature_flag_enabled=True, - feature_flag_refresh_enabled=True, # Is it needed? - refresh_interval=60, # wait at least 60s before next refresh + feature_flag_refresh_enabled=True ) feature_manager = FeatureManager(app_config, on_feature_evaluated=publish_telemetry) app.state.app_config = app_config diff --git a/src/api/routes.py b/src/api/routes.py index 8d430bd..ea58751 100644 --- a/src/api/routes.py +++ b/src/api/routes.py @@ -231,8 +231,8 @@ async def chat( app_config = Depends(get_app_config), ): # Refresh config if configured - #if app_config: - #app_config.refresh() + if app_config: + app_config.refresh() # Retrieve the thread ID from the cookies (if available). thread_id = request.cookies.get('thread_id') @@ -251,9 +251,8 @@ async def chat( raise HTTPException(status_code=400, detail=f"Error handling thread: {e}") thread_id = thread.id - #agent_id = await get_agent_variant(feature_manager, ai_client) or agent.id - agent_id = agent.id - + agent_id = await get_agent_variant(feature_manager, ai_client) or agent.id + # Parse the JSON from the request. try: user_message = await request.json() diff --git a/src/requirements.txt b/src/requirements.txt index 0084f0e..184d86a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -3,14 +3,14 @@ uvicorn[standard]==0.29.0 gunicorn==22.0.0 azure-identity==1.19.0 aiohttp==3.11.1 -azure-ai-projects==1.0.0b7 +azure-ai-projects==1.0.0b8 azure-core-tracing-opentelemetry azure-monitor-opentelemetry azure-search-documents opentelemetry-sdk azure-ai-evaluation -azure-appconfiguration-provider==2.0.0b3 +azure-appconfiguration-provider==2.1.0b1 FeatureManagement==2.0.0b3 azure-monitor-opentelemetry-exporter azure-monitor-events-extension \ No newline at end of file From f8bb98c801c0f8ecbfacf258ef66125dbe622cca Mon Sep 17 00:00:00 2001 From: aprilk-ms <55356546+aprilk-ms@users.noreply.github.com> Date: Mon, 14 Apr 2025 05:17:53 +0000 Subject: [PATCH 11/30] Add variant to message title --- src/api/routes.py | 15 +++++++++------ src/api/static/ChatClient.js | 14 ++++++++------ src/api/static/ChatUI.js | 12 ++++++++++-- src/api/templates/index.html | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/api/routes.py b/src/api/routes.py index ea58751..6fac214 100644 --- a/src/api/routes.py +++ b/src/api/routes.py @@ -139,16 +139,16 @@ async def index(request: Request): return templates.TemplateResponse("index.html", {"request": request}) -async def get_agent_variant(feature_manager, ai_client: AIProjectClient) -> str: +async def get_agent_variant(feature_manager, ai_client: AIProjectClient): if feature_manager: agent_variant = feature_manager.get_variant("my-agent") if agent_variant and agent_variant.configuration: try: - agent_variant = await ai_client.agents.get_agent(agent_variant.configuration) - logger.info(f"Using agent variant: {agent_variant.id}") - return agent_variant.id + agent = await ai_client.agents.get_agent(agent_variant.configuration) + logger.info(f"Using agent variant: {agent.id}") + return agent_variant except Exception as e: - logger.error(f"Error retrieving agent variant with Id {agent_variant.id}. {e}") + logger.error(f"Error retrieving agent variant with Id {agent_variant.configuration}. {e}") return None async def get_result(thread_id: str, agent_id: str, ai_client : AIProjectClient) -> AsyncGenerator[str, None]: @@ -251,7 +251,8 @@ async def chat( raise HTTPException(status_code=400, detail=f"Error handling thread: {e}") thread_id = thread.id - agent_id = await get_agent_variant(feature_manager, ai_client) or agent.id + agent_variant = await get_agent_variant(feature_manager, ai_client) + agent_id = agent_variant.configuration if agent_variant else agent.id # Parse the JSON from the request. try: @@ -288,6 +289,8 @@ async def chat( # Update cookies to persist the thread and agent IDs. response.set_cookie("thread_id", thread_id) response.set_cookie("agent_id", agent_id) + + response.headers["agent-variant"] = str(agent_variant.name) if agent_variant else None return response diff --git a/src/api/static/ChatClient.js b/src/api/static/ChatClient.js index 5e018aa..8851719 100644 --- a/src/api/static/ChatClient.js +++ b/src/api/static/ChatClient.js @@ -45,8 +45,10 @@ class ChatClient { throw new Error('ReadableStream not supported or response.body is null'); } + const agentVariant = response.headers.get('agent-variant'); + console.log("[ChatClient] Starting to handle streaming response..."); - this.handleMessages(response.body); + this.handleMessages(response.body, agentVariant); } catch (error) { document.getElementById("generating-message").style.display = "none"; @@ -61,12 +63,12 @@ class ChatClient { } } - handleMessages(stream) { + handleMessages(stream, agentVariant) { let messageDiv = null; let accumulatedContent = ''; let isStreaming = true; let buffer = ''; - let annotations = []; + let annotations = []; // Create a reader for the SSE stream const reader = stream.getReader(); @@ -110,7 +112,7 @@ class ChatClient { if (data.error) { if (!messageDiv) { - messageDiv = this.ui.createAssistantMessageDiv(); + messageDiv = this.ui.createAssistantMessageDiv(agentVariant); console.log("[ChatClient] Created new messageDiv for assistant."); } document.getElementById("generating-message").style.display = "none"; @@ -119,7 +121,7 @@ class ChatClient { data.error.message || "An error occurred.", false ); - return; + return; } // Check the data type to decide how to update the UI @@ -135,7 +137,7 @@ class ChatClient { } else { // If we have no messageDiv yet, create one if (!messageDiv) { - messageDiv = this.ui.createAssistantMessageDiv(); + messageDiv = this.ui.createAssistantMessageDiv(agentVariant); console.log("[ChatClient] Created new messageDiv for assistant."); } diff --git a/src/api/static/ChatUI.js b/src/api/static/ChatUI.js index f957dd3..dd80a7d 100644 --- a/src/api/static/ChatUI.js +++ b/src/api/static/ChatUI.js @@ -175,7 +175,7 @@ class ChatUI { } } - createAssistantMessageDiv() { + createAssistantMessageDiv(agentVariant) { const assistantTemplateClone = this.assistantTemplate.content.cloneNode(true); if (!assistantTemplateClone) { console.error("Failed to clone assistant template."); @@ -184,7 +184,7 @@ class ChatUI { // Remove the placeholder message this.removePlaceholder(); - + // Append the clone to the target container this.targetContainer.appendChild(assistantTemplateClone); @@ -204,6 +204,14 @@ class ChatUI { if (!messageDiv) { console.error("Message content div not found in the template."); } + + // Update message title if a variant is used + if (agentVariant) { + const messageTitleDiv = newlyAddedToast.querySelector(".message-title"); + if (messageTitleDiv) { + messageTitleDiv.innerHTML += ` (Variant: ${agentVariant})`; + } + } return messageDiv; } diff --git a/src/api/templates/index.html b/src/api/templates/index.html index a813f43..d79d85a 100644 --- a/src/api/templates/index.html +++ b/src/api/templates/index.html @@ -92,7 +92,7 @@
Document Viewer