diff --git a/.github/workflows/deploy-v2.yml b/.github/workflows/deploy-v2.yml new file mode 100644 index 00000000..58b0e3a1 --- /dev/null +++ b/.github/workflows/deploy-v2.yml @@ -0,0 +1,346 @@ +name: Validate Deployment v2 + +on: + workflow_run: + workflows: ["Build Docker and Optional Push"] + types: + - completed + branches: + - macae-v2 + - dev + schedule: + - cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT + workflow_dispatch: #Allow manual triggering +env: + GPT_MIN_CAPACITY: 150 + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + deploy: + runs-on: ubuntu-latest + outputs: + RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }} + WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }} + DEPLOYMENT_SUCCESS: ${{ steps.deployment_status.outputs.SUCCESS }} + MACAE_URL_API: ${{ steps.get_backend_url.outputs.MACAE_URL_API }} + CONTAINER_APP: ${{steps.get_backend_url.outputs.CONTAINER_APP}} + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Run Quota Check + id: quota-check + run: | + export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }} + export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} + export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} + export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" + export GPT_MIN_CAPACITY="150" + export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" + + chmod +x infra/scripts/checkquota.sh + if ! infra/scripts/checkquota.sh; then + # If quota check fails due to insufficient quota, set the flag + if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then + echo "QUOTA_FAILED=true" >> $GITHUB_ENV + fi + exit 1 # Fail the pipeline if any other failure occurs + fi + + - name: Send Notification on Quota Failure + if: env.QUOTA_FAILED == 'true' + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + EMAIL_BODY=$(cat <Dear Team,

The quota check has failed, and the pipeline cannot proceed.

Build URL: ${RUN_URL}

Please take necessary action.

Best regards,
Your Automation Team

" + } + EOF + ) + + curl -X POST "${{ secrets.AUTO_LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + - name: Fail Pipeline if Quota Check Fails + if: env.QUOTA_FAILED == 'true' + run: exit 1 + + - name: Set Deployment Region + run: | + echo "Selected Region: $VALID_REGION" + echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV + + - name: Setup Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version # Verify installation + + - name: Login to Azure + run: | + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + + - name: Install Bicep CLI + run: az bicep install + + - name: Generate Resource Group Name + id: generate_rg_name + run: | + ACCL_NAME="macae" + SHORT_UUID=$(uuidgen | cut -d'-' -f1) + UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}" + echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV + echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}" + + - name: Check and Create Resource Group + id: check_create_rg + run: | + set -e + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "false" ]; then + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} + fi + echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT + + - name: Generate Unique Solution Prefix + id: generate_solution_prefix + run: | + COMMON_PART="macae" + TIMESTAMP=$(date +%s) + UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6) + UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}" + echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV + + - name: Deploy Bicep Template + id: deploy + run: | + if [[ "${{ env.BRANCH_NAME }}" == "macae-v2" ]]; then + IMAGE_TAG="latest" + elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then + IMAGE_TAG="dev" + # elif [[ "${{ env.BRANCH_NAME }}" == "hotfix" ]]; then + # IMAGE_TAG="hotfix" + # else + # IMAGE_TAG="latest" + fi + + az deployment group create \ + --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ + --template-file infra/main.bicep \ + --parameters \ + solutionName=${{ env.SOLUTION_PREFIX }} \ + location="${{ env.AZURE_LOCATION }}" \ + gptModelDeploymentType="GlobalStandard" \ + gptModelName="gpt-4o" \ + gptModelVersion="2024-08-06" \ + backendContainerImageTag="${IMAGE_TAG}" \ + frontendContainerImageTag="${IMAGE_TAG}" \ + azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \ + gptModelCapacity=150 \ + createdBy="Pipeline" \ + --output json + + - name: Extract Web App and API App URLs + id: get_output + run: | + WEBAPP_NAMES=$(az webapp list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[].name" -o tsv) + for NAME in $WEBAPP_NAMES; do + if [[ $NAME == app-* ]]; then + WEBAPP_URL="https://${NAME}.azurewebsites.net" + echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT + fi + done + + - name: Get Container App Backend URL + id: get_backend_url + run: | + CONTAINER_APP_NAME=$(az containerapp list \ + --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ + --query "[0].name" -o tsv) + + MACAE_URL_API=$(az containerapp show \ + --name "$CONTAINER_APP_NAME" \ + --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ + --query "properties.configuration.ingress.fqdn" -o tsv) + + echo "MACAE_URL_API=https://${MACAE_URL_API}" >> $GITHUB_OUTPUT + echo "CONTAINER_APP=${CONTAINER_APP_NAME}" >> $GITHUB_OUTPUT + + - name: Set Deployment Status + id: deployment_status + if: always() + run: | + if [ "${{ job.status }}" == "success" ]; then + echo "SUCCESS=true" >> $GITHUB_OUTPUT + else + echo "SUCCESS=false" >> $GITHUB_OUTPUT + fi + + e2e-test: + needs: deploy + if: needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true' + uses: ./.github/workflows/test-automation.yml + with: + MACAE_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }} + MACAE_URL_API: ${{ needs.deploy.outputs.MACAE_URL_API }} + MACAE_RG: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }} + MACAE_CONTAINER_APP: ${{ needs.deploy.outputs.CONTAINER_APP }} + secrets: inherit + + cleanup-deployment: + if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != '' + needs: [deploy, e2e-test] + runs-on: ubuntu-latest + env: + RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }} + steps: + - name: Setup Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version + - name: Login to Azure + run: | + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" + + - name: Extract AI Services and Key Vault Names + if: always() + run: | + echo "Fetching AI Services and Key Vault names before deletion..." + + # Get Key Vault name + KEYVAULT_NAME=$(az resource list --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --resource-type "Microsoft.KeyVault/vaults" --query "[].name" -o tsv) + echo "Detected Key Vault: $KEYVAULT_NAME" + echo "KEYVAULT_NAME=$KEYVAULT_NAME" >> $GITHUB_ENV + # Extract AI Services names + echo "Fetching AI Services..." + AI_SERVICES=$(az resource list --resource-group '${{ env.RESOURCE_GROUP_NAME }}' --resource-type "Microsoft.CognitiveServices/accounts" --query "[].name" -o tsv) + # Flatten newline-separated values to space-separated + AI_SERVICES=$(echo "$AI_SERVICES" | paste -sd ' ' -) + echo "Detected AI Services: $AI_SERVICES" + echo "AI_SERVICES=$AI_SERVICES" >> $GITHUB_ENV + + - name: Get OpenAI Resource from Resource Group + id: get_openai_resource + run: | + + set -e + echo "Fetching OpenAI resource from resource group ${{ env.RESOURCE_GROUP_NAME }}..." + + # Run the az resource list command to get the OpenAI resource name + openai_resource_name=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].name" -o tsv) + + if [ -z "$openai_resource_name" ]; then + echo "No OpenAI resource found in resource group ${{ env.RESOURCE_GROUP_NAME }}." + exit 0 + else + echo "OPENAI_RESOURCE_NAME=${openai_resource_name}" >> $GITHUB_ENV + echo "OpenAI resource name: ${openai_resource_name}" + fi + + - name: Delete Bicep Deployment + if: always() + run: | + set -e + echo "Checking if resource group exists..." + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "true" ]; then + echo "Resource group exist. Cleaning..." + az group delete \ + --name ${{ env.RESOURCE_GROUP_NAME }} \ + --yes \ + --no-wait + echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}" + else + echo "Resource group does not exists." + fi + + - name: Wait for resource deletion to complete + run: | + + # Add resources to the array + resources_to_check=("${{ env.OPENAI_RESOURCE_NAME }}") + + echo "List of resources to check: ${resources_to_check[@]}" + + # Maximum number of retries + max_retries=3 + + # Retry intervals in seconds (30, 60, 120) + retry_intervals=(30 60 120) + + # Retry mechanism to check resources + retries=0 + while true; do + resource_found=false + + # Get the list of resources in YAML format again on each retry + resource_list=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --output yaml) + + # Iterate through the resources to check + for resource in "${resources_to_check[@]}"; do + echo "Checking resource: $resource" + if echo "$resource_list" | grep -q "name: $resource"; then + echo "Resource '$resource' exists in the resource group." + resource_found=true + else + echo "Resource '$resource' does not exist in the resource group." + fi + done + + # If any resource exists, retry + if [ "$resource_found" = true ]; then + retries=$((retries + 1)) + if [ "$retries" -gt "$max_retries" ]; then + echo "Maximum retry attempts reached. Exiting." + break + else + # Wait for the appropriate interval for the current retry + echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..." + sleep ${retry_intervals[$retries-1]} + fi + else + echo "No resources found. Exiting." + break + fi + done + + - name: Purging the Resources + if: always() + run: | + + set -e + echo "Azure OpenAI: ${{ env.OPENAI_RESOURCE_NAME }}" + + # Purge OpenAI Resource + echo "Purging the OpenAI Resource..." + if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then + echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" + else + echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" + fi + + echo "Resource purging completed successfully" + + - name: Send Notification on Failure + if: failure() + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + # Construct the email body + EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the Multi-Agent-Custom-Automation-Engine-Solution-Accelerator Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}
${OUTPUT}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + } + EOF + ) + + # Send the notification + curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + - name: Logout from Azure + if: always() + run: | + az logout + echo "Logged out from Azure." diff --git a/.github/workflows/deploy-waf-v2.yml b/.github/workflows/deploy-waf-v2.yml new file mode 100644 index 00000000..1726747b --- /dev/null +++ b/.github/workflows/deploy-waf-v2.yml @@ -0,0 +1,243 @@ +name: Validate WAF Deployment v2 + +on: + push: + branches: + - macae-v2 + schedule: + - cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Run Quota Check + id: quota-check + run: | + export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }} + export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} + export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} + export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" + export GPT_MIN_CAPACITY="150" + export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" + + chmod +x infra/scripts/checkquota.sh + if ! infra/scripts/checkquota.sh; then + # If quota check fails due to insufficient quota, set the flag + if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then + echo "QUOTA_FAILED=true" >> $GITHUB_ENV + fi + exit 1 # Fail the pipeline if any other failure occurs + fi + + - name: Send Notification on Quota Failure + if: env.QUOTA_FAILED == 'true' + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + EMAIL_BODY=$(cat <Dear Team,

The quota check has failed, and the pipeline cannot proceed.

Build URL: ${RUN_URL}

Please take necessary action.

Best regards,
Your Automation Team

" + } + EOF + ) + + curl -X POST "${{ secrets.AUTO_LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + - name: Fail Pipeline if Quota Check Fails + if: env.QUOTA_FAILED == 'true' + run: exit 1 + + - name: Set Deployment Region + run: | + echo "Selected Region: $VALID_REGION" + echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV + + - name: Setup Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version # Verify installation + + - name: Login to Azure + run: | + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + + - name: Install Bicep CLI + run: az bicep install + + - name: Generate Resource Group Name + id: generate_rg_name + run: | + echo "Generating a unique resource group name..." + ACCL_NAME="macae" # Account name as specified + SHORT_UUID=$(uuidgen | cut -d'-' -f1) + UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}" + echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV + echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}" + + - name: Check and Create Resource Group + id: check_create_rg + run: | + set -e + echo "Checking if resource group exists..." + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "false" ]; then + echo "Resource group does not exist. Creating..." + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location ${{ env.AZURE_LOCATION }} || { echo "Error creating resource group"; exit 1; } + else + echo "Resource group already exists." + fi + + - name: Generate Unique Solution Prefix + id: generate_solution_prefix + run: | + COMMON_PART="macae" + TIMESTAMP=$(date +%s) + UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 6) + UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}" + echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV + + - name: Deploy Bicep Template + id: deploy + run: | + set -e + az deployment group create \ + --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ + --template-file infra/main.bicep \ + --parameters \ + solutionName=${{ env.SOLUTION_PREFIX }} \ + location="${{ env.AZURE_LOCATION }}" \ + azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \ + gptModelCapacity=5 \ + enableTelemetry=true \ + enableMonitoring=true \ + enablePrivateNetworking=true \ + enableScalability=true \ + createdBy="Pipeline" \ + + + - name: Send Notification on Failure + if: failure() + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + # Construct the email body + EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the Multi-Agent-Custom-Automation-Engine-Solution-Accelerator Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}
${OUTPUT}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + } + EOF + ) + + # Send the notification + curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + - name: Get OpenAI Resource from Resource Group + id: get_openai_resource + run: | + + + set -e + echo "Fetching OpenAI resource from resource group ${{ env.RESOURCE_GROUP_NAME }}..." + + # Run the az resource list command to get the OpenAI resource name + openai_resource_name=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].name" -o tsv) + + if [ -z "$openai_resource_name" ]; then + echo "No OpenAI resource found in resource group ${{ env.RESOURCE_GROUP_NAME }}." + exit 1 + else + echo "OPENAI_RESOURCE_NAME=${openai_resource_name}" >> $GITHUB_ENV + echo "OpenAI resource name: ${openai_resource_name}" + fi + + - name: Delete Bicep Deployment + if: always() + run: | + set -e + echo "Checking if resource group exists..." + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "true" ]; then + echo "Resource group exist. Cleaning..." + az group delete \ + --name ${{ env.RESOURCE_GROUP_NAME }} \ + --yes \ + --no-wait + echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}" + else + echo "Resource group does not exists." + fi + + - name: Wait for resource deletion to complete + run: | + + + # Add resources to the array + resources_to_check=("${{ env.OPENAI_RESOURCE_NAME }}") + + echo "List of resources to check: ${resources_to_check[@]}" + + # Maximum number of retries + max_retries=3 + + # Retry intervals in seconds (30, 60, 120) + retry_intervals=(30 60 120) + + # Retry mechanism to check resources + retries=0 + while true; do + resource_found=false + + # Get the list of resources in YAML format again on each retry + resource_list=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --output yaml) + + # Iterate through the resources to check + for resource in "${resources_to_check[@]}"; do + echo "Checking resource: $resource" + if echo "$resource_list" | grep -q "name: $resource"; then + echo "Resource '$resource' exists in the resource group." + resource_found=true + else + echo "Resource '$resource' does not exist in the resource group." + fi + done + + # If any resource exists, retry + if [ "$resource_found" = true ]; then + retries=$((retries + 1)) + if [ "$retries" -gt "$max_retries" ]; then + echo "Maximum retry attempts reached. Exiting." + break + else + # Wait for the appropriate interval for the current retry + echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..." + sleep ${retry_intervals[$retries-1]} + fi + else + echo "No resources found. Exiting." + break + fi + done + + - name: Purging the Resources + if: always() + run: | + + set -e + echo "Azure OpenAI: ${{ env.OPENAI_RESOURCE_NAME }}" + + # Purge OpenAI Resource + echo "Purging the OpenAI Resource..." + if ! az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/eastus/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose; then + echo "Failed to purge openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" + else + echo "Purged the openai resource: ${{ env.OPENAI_RESOURCE_NAME }}" + fi + + echo "Resource purging completed successfully" diff --git a/.github/workflows/deploy-waf.yml b/.github/workflows/deploy-waf.yml index e2786216..eb0e5a61 100644 --- a/.github/workflows/deploy-waf.yml +++ b/.github/workflows/deploy-waf.yml @@ -1,4 +1,4 @@ -name: Validate WAF Deployment +name: Validate WAF Deployment v3 on: push: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 71770955..026b9424 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,4 +1,4 @@ -name: Validate Deployment +name: Validate Deployment v3 on: workflow_run: @@ -7,8 +7,8 @@ on: - completed branches: - main + - dev-v3 - hotfix - - dev schedule: - cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT workflow_dispatch: #Allow manual triggering @@ -116,9 +116,9 @@ jobs: id: deploy run: | if [[ "${{ env.BRANCH_NAME }}" == "main" ]]; then - IMAGE_TAG="latest" - elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then - IMAGE_TAG="dev" + IMAGE_TAG="latest_v3" + elif [[ "${{ env.BRANCH_NAME }}" == "dev-v3" ]]; then + IMAGE_TAG="dev_v3" elif [[ "${{ env.BRANCH_NAME }}" == "hotfix" ]]; then IMAGE_TAG="hotfix" else diff --git a/.github/workflows/docker-build-and-push-v2.yml b/.github/workflows/docker-build-and-push-v2.yml new file mode 100644 index 00000000..3d08734f --- /dev/null +++ b/.github/workflows/docker-build-and-push-v2.yml @@ -0,0 +1,95 @@ +name: Build Docker and Optional Push v2 + +on: + push: + branches: + # - main + - dev + # - dev-v3 + - macae-v2 + - demo + # - hotfix + pull_request: + types: + - opened + - ready_for_review + - reopened + - synchronize + branches: + # - main + - dev + # - dev-v3 + - macae-v2 + - demo + # - hotfix + workflow_dispatch: + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Log in to Azure Container Registry + if: ${{ github.ref_name == 'macae-v2' || github.ref_name == 'dev' || github.ref_name == 'demo'}} + uses: azure/docker-login@v2 + with: + login-server: ${{ secrets.ACR_LOGIN_SERVER || 'acrlogin.azurecr.io' }} + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Get registry + id: registry + run: | + echo "ext_registry=${{ secrets.ACR_LOGIN_SERVER || 'acrlogin.azurecr.io'}}" >> $GITHUB_OUTPUT + + - name: Determine Tag Name Based on Branch + id: determine_tag + run: | + + if [[ "${{ github.ref }}" == "refs/heads/macae-v2" ]]; then + echo "TAG=latest" >> $GITHUB_ENV + elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then + echo "TAG=dev" >> $GITHUB_ENV + elif [[ "${{ github.ref }}" == "refs/heads/demo" ]]; then + echo "TAG=demo" >> $GITHUB_ENV + else + echo "TAG=pullrequest-ignore" >> $GITHUB_ENV + fi + + - name: Set Historical Tag + run: | + DATE_TAG=$(date +'%Y-%m-%d') + RUN_ID=${{ github.run_number }} + # Create historical tag using TAG, DATE_TAG, and RUN_ID + echo "HISTORICAL_TAG=${{ env.TAG }}_${DATE_TAG}_${RUN_ID}" >> $GITHUB_ENV + + - name: Build and optionally push Backend Docker image + uses: docker/build-push-action@v6 + with: + context: ./src/backend + file: ./src/backend/Dockerfile + push: ${{ env.TAG != 'pullrequest-ignore' }} + tags: | + ${{ steps.registry.outputs.ext_registry }}/macaebackend:${{ env.TAG }} + ${{ steps.registry.outputs.ext_registry }}/macaebackend:${{ env.HISTORICAL_TAG }} + + - name: Build and optionally push Frontend Docker image + uses: docker/build-push-action@v6 + with: + context: ./src/frontend + file: ./src/frontend/Dockerfile + push: ${{ env.TAG != 'pullrequest-ignore' }} + tags: | + ${{ steps.registry.outputs.ext_registry }}/macaefrontend:${{ env.TAG }} + ${{ steps.registry.outputs.ext_registry }}/macaefrontend:${{ env.HISTORICAL_TAG }} + \ No newline at end of file diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index 359320d5..eb46863e 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -1,11 +1,11 @@ -name: Build Docker and Optional Push +name: Build Docker and Optional Push v3 on: push: branches: - main - - dev - - demo + - dev-v3 + - demo-v3 - hotfix pull_request: types: @@ -15,8 +15,8 @@ on: - synchronize branches: - main - - dev - - demo + - dev-v3 + - demo-v3 - hotfix workflow_dispatch: @@ -32,7 +32,7 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Log in to Azure Container Registry - if: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' }} + if: ${{ github.ref_name == 'main' || github.ref_name == 'dev-v3'|| github.ref_name == 'demo-v3' || github.ref_name == 'hotfix' }} uses: azure/docker-login@v2 with: login-server: ${{ secrets.ACR_LOGIN_SERVER || 'acrlogin.azurecr.io' }} @@ -52,11 +52,11 @@ jobs: id: determine_tag run: | if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "TAG=latest" >> $GITHUB_ENV - elif [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then - echo "TAG=dev" >> $GITHUB_ENV - elif [[ "${{ github.ref }}" == "refs/heads/demo" ]]; then - echo "TAG=demo" >> $GITHUB_ENV + echo "TAG=latest_v3" >> $GITHUB_ENV + elif [[ "${{ github.ref }}" == "refs/heads/dev-v3" ]]; then + echo "TAG=dev_v3" >> $GITHUB_ENV + elif [[ "${{ github.ref }}" == "refs/heads/demo-v3" ]]; then + echo "TAG=demo_v3" >> $GITHUB_ENV elif [[ "${{ github.ref }}" == "refs/heads/hotfix" ]]; then echo "TAG=hotfix" >> $GITHUB_ENV else @@ -88,4 +88,14 @@ jobs: push: ${{ env.TAG != 'pullrequest-ignore' }} tags: | ${{ steps.registry.outputs.ext_registry }}/macaefrontend:${{ env.TAG }} - ${{ steps.registry.outputs.ext_registry }}/macaefrontend:${{ env.HISTORICAL_TAG }} \ No newline at end of file + ${{ steps.registry.outputs.ext_registry }}/macaefrontend:${{ env.HISTORICAL_TAG }} + + - name: Build and optionally push MCP Docker image + uses: docker/build-push-action@v6 + with: + context: ./src/mcp_server + file: ./src/mcp_server/Dockerfile + push: ${{ env.TAG != 'pullrequest-ignore' }} + tags: | + ${{ steps.registry.outputs.ext_registry }}/macaemcp:${{ env.TAG }} + ${{ steps.registry.outputs.ext_registry }}/macaemcp:${{ env.HISTORICAL_TAG }} \ No newline at end of file diff --git a/docs/docker_mcp_server_testing.md b/docs/docker_mcp_server_testing.md index dd5c13c4..62d41b65 100644 --- a/docs/docker_mcp_server_testing.md +++ b/docs/docker_mcp_server_testing.md @@ -402,4 +402,4 @@ docker inspect --format='{{.State.Health.Status}}' macae-mcp-server --- -For additional help and troubleshooting, refer to the main [DeploymentGuide.md](./DeploymentGuide.md) and [LocalDeployment.md](./LocalDeployment.md) documentation. +For additional help and troubleshooting, refer to the main [DeploymentGuide.md](./DeploymentGuide.md) documentation. diff --git a/infra/main.bicep b/infra/main.bicep index 284d3cdf..1b409557 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -132,31 +132,31 @@ param virtualMachineAdminPassword string = newGuid() // These parameters are changed for testing - please reset as part of publication @description('Optional. The Container Registry hostname where the docker images for the backend are located.') -param backendContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param backendContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the backend.') -param backendContainerImageName string = 'macae-backend' +param backendContainerImageName string = 'macaebackend' @description('Optional. The Container Image Tag to deploy on the backend.') -param backendContainerImageTag string = 'v3tst1' +param backendContainerImageTag string = 'latest_v3' @description('Optional. The Container Registry hostname where the docker images for the frontend are located.') -param frontendContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param frontendContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the frontend.') -param frontendContainerImageName string = 'macae-frontend' +param frontendContainerImageName string = 'macaefrontend' @description('Optional. The Container Image Tag to deploy on the frontend.') -param frontendContainerImageTag string = 'v3tst1' +param frontendContainerImageTag string = 'latest_v3' @description('Optional. The Container Registry hostname where the docker images for the MCP are located.') -param MCPContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param MCPContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the MCP.') -param MCPContainerImageName string = 'mcp_server' +param MCPContainerImageName string = 'macaemcp' @description('Optional. The Container Image Tag to deploy on the MCP.') -param MCPContainerImageTag string = 'v3tst1' +param MCPContainerImageTag string = 'latest_v3' @description('Optional. Enable/Disable usage telemetry for module.') param enableTelemetry bool = true @@ -726,6 +726,7 @@ module bastionHost 'br/public:avm/res/network/bastion-host:0.7.0' = if (enablePr enableTelemetry: enableTelemetry tags: tags virtualNetworkResourceId: virtualNetwork!.?outputs.?resourceId + availabilityZones:[] publicIPAddressObject: { name: 'pip-bas${solutionSuffix}' diagnosticSettings: enableMonitoring ? [{ workspaceResourceId: logAnalyticsWorkspaceResourceId }] : null @@ -1499,7 +1500,7 @@ module containerApp 'br/public:avm/res/app/container-app:0.18.1' = { // WAF aligned configuration for Scalability scaleSettings: { maxReplicas: enableScalability ? 3 : 1 - minReplicas: enableScalability ? 2 : 1 + minReplicas: enableScalability ? 1 : 1 rules: [ { name: 'http-scaler' @@ -1691,7 +1692,7 @@ module containerAppMcp 'br/public:avm/res/app/container-app:0.18.1' = { // WAF aligned configuration for Scalability scaleSettings: { maxReplicas: enableScalability ? 3 : 1 - minReplicas: enableScalability ? 2 : 1 + minReplicas: enableScalability ? 1 : 1 rules: [ { name: 'http-scaler' diff --git a/infra/main.parameters.json b/infra/main.parameters.json index fde05cc8..7e0ffe4e 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -24,10 +24,13 @@ "value": "${AZURE_ENV_MODEL_CAPACITY}" }, "backendContainerImageTag": { - "value": "${AZURE_ENV_IMAGE_TAG=v3tst1}" + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" }, "frontendContainerImageTag": { - "value": "${AZURE_ENV_IMAGE_TAG=v3tst1}" + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" + }, + "MCPContainerImageTag": { + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" }, "enableTelemetry": { "value": "${AZURE_ENV_ENABLE_TELEMETRY}" diff --git a/infra/main.waf.parameters.json b/infra/main.waf.parameters.json index 67a9916c..51d451d5 100644 --- a/infra/main.waf.parameters.json +++ b/infra/main.waf.parameters.json @@ -24,10 +24,13 @@ "value": "${AZURE_ENV_MODEL_CAPACITY}" }, "backendContainerImageTag": { - "value": "${AZURE_ENV_IMAGE_TAG=latest}" + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" }, "frontendContainerImageTag": { - "value": "${AZURE_ENV_IMAGE_TAG=latest}" + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" + }, + "MCPContainerImageTag": { + "value": "${AZURE_ENV_IMAGE_TAG=latest_v3}" }, "enableTelemetry": { "value": "${AZURE_ENV_ENABLE_TELEMETRY}" diff --git a/infra/main_custom.bicep b/infra/main_custom.bicep index 6c4bf032..021073c4 100644 --- a/infra/main_custom.bicep +++ b/infra/main_custom.bicep @@ -133,31 +133,31 @@ param virtualMachineAdminPassword string = newGuid() // These parameters are changed for testing - please reset as part of publication @description('Optional. The Container Registry hostname where the docker images for the backend are located.') -param backendContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param backendContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the backend.') -param backendContainerImageName string = 'macae-backend' +param backendContainerImageName string = 'macaebackend' @description('Optional. The Container Image Tag to deploy on the backend.') -param backendContainerImageTag string = 'v3tst1' +param backendContainerImageTag string = 'latest_v3' @description('Optional. The Container Registry hostname where the docker images for the frontend are located.') -param frontendContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param frontendContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the frontend.') -param frontendContainerImageName string = 'macae-frontend' +param frontendContainerImageName string = 'macaefrontend' @description('Optional. The Container Image Tag to deploy on the frontend.') -param frontendContainerImageTag string = 'v3tst1' +param frontendContainerImageTag string = 'latest_v3' @description('Optional. The Container Registry hostname where the docker images for the MCP are located.') -param MCPContainerRegistryHostname string = 'macaev3tst1acr.azurecr.io' +param MCPContainerRegistryHostname string = 'biabcontainerreg.azurecr.io' @description('Optional. The Container Image Name to deploy on the MCP.') -param MCPContainerImageName string = 'mcp_server' +param MCPContainerImageName string = 'macaemcp' @description('Optional. The Container Image Tag to deploy on the MCP.') -param MCPContainerImageTag string = 'v3tst1' +param MCPContainerImageTag string = 'latest_v3' @description('Optional. Enable/Disable usage telemetry for module.') param enableTelemetry bool = true @@ -284,7 +284,7 @@ module logAnalyticsWorkspace 'br/public:avm/res/operational-insights/workspace:0 features: { enableLogAccessUsingOnlyResourcePermissions: true } diagnosticSettings: [{ useThisWorkspace: true }] // WAF aligned configuration for Redundancy - dailyQuotaGb: enableRedundancy ? 10 : null //WAF recommendation: 10 GB per day is a good starting point for most workloads + dailyQuotaGb: enableRedundancy ? 150 : null //WAF recommendation: 150 GB per day is a good starting point for most workloads replication: enableRedundancy ? { enabled: true @@ -725,6 +725,7 @@ module bastionHost 'br/public:avm/res/network/bastion-host:0.7.0' = if (enablePr enableTelemetry: enableTelemetry tags: tags virtualNetworkResourceId: virtualNetwork!.?outputs.?resourceId + availabilityZones:[] publicIPAddressObject: { name: 'pip-bas${solutionSuffix}' diagnosticSettings: enableMonitoring ? [{ workspaceResourceId: logAnalyticsWorkspaceResourceId }] : null @@ -849,6 +850,26 @@ module windowsVmDataCollectionRules 'br/public:avm/res/insights/data-collection- name: 'perfCounterDataSource60' } ] + windowsEventLogs: [ + { + name: 'SecurityAuditEvents' + streams: [ + 'Microsoft-WindowsEvent' + ] + eventLogName: 'Security' + eventTypes: [ + { + eventType: 'Audit Success' + } + { + eventType: 'Audit Failure' + } + ] + xPathQueries: [ + 'Security!*[System[(EventID=4624 or EventID=4625)]]' + ] + } + ] } destinations: { logAnalytics: [ @@ -906,7 +927,7 @@ module virtualMachine 'br/public:avm/res/compute/virtual-machine:0.17.0' = if (e bypassPlatformSafetyChecksOnUserSchedule: true maintenanceConfigurationResourceId: maintenanceConfiguration!.outputs.resourceId enableAutomaticUpdates: true - encryptionAtHost: false + encryptionAtHost: true availabilityZone: virtualMachineAvailabilityZone proximityPlacementGroupResourceId: proximityPlacementGroup!.outputs.resourceId imageReference: { @@ -1504,7 +1525,7 @@ module containerApp 'br/public:avm/res/app/container-app:0.18.1' = { // WAF aligned configuration for Scalability scaleSettings: { maxReplicas: enableScalability ? 3 : 1 - minReplicas: enableScalability ? 2 : 1 + minReplicas: enableScalability ? 1 : 1 rules: [ { name: 'http-scaler' @@ -1702,7 +1723,7 @@ module containerAppMcp 'br/public:avm/res/app/container-app:0.18.1' = { // WAF aligned configuration for Scalability scaleSettings: { maxReplicas: enableScalability ? 3 : 1 - minReplicas: enableScalability ? 2 : 1 + minReplicas: enableScalability ? 1 : 1 rules: [ { name: 'http-scaler' @@ -1845,6 +1866,7 @@ module webSite 'modules/web-sites.bicep' = { vnetImagePullEnabled: enablePrivateNetworking ? true : false virtualNetworkSubnetId: enablePrivateNetworking ? virtualNetwork!.outputs.subnetResourceIds[4] : null publicNetworkAccess: 'Enabled' // Always enabling the public network access for Web App + e2eEncryptionEnabled: true } } diff --git a/src/mcp_server/config/settings.py b/src/mcp_server/config/settings.py index 58e96634..85f78b96 100644 --- a/src/mcp_server/config/settings.py +++ b/src/mcp_server/config/settings.py @@ -31,7 +31,7 @@ class MCPServerConfig(BaseSettings): audience: Optional[str] = Field(default=None) # MCP specific settings - server_name: str = Field(default="MACAE MCP Server") + server_name: str = Field(default="MacaeMcpServer") enable_auth: bool = Field(default=True) # Dataset path - added to handle the environment variable