Integrate Azure Functions deployment with In-Vitro

.github/workflows/azure_integration_test.yaml

@@ -0,0 +1,65 @@
+name: Azure Integration Tests
+
+on:
+  push:
+    branches:
+      - main # Trigger the workflow when code is pushed to the main branch
+  pull_request:
+    branches:
+      - main # Trigger the workflow when the PR targets the main branch
+  workflow_dispatch: # Allows manual triggering of the workflow
+
+env:
+  GOOS: linux
+  GO111MODULE: on
+
+jobs:
+  test-azure:
+    name: Run Azure Integration Tests
+    runs-on: ubuntu-24.04
+    env:
+      AZURE_APP_ID: ${{ secrets.AZURE_APP_ID }}
+      AZURE_PASSWORD: ${{ secrets.AZURE_PASSWORD }}
+      AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
+
+    steps:
+      - name: Check if environment variables are set  # Validate secrets are passed
+        run: |
+          if [[ -z "$AZURE_APP_ID" ]]; then
+            echo "AZURE_APP_ID is not set. Please check if secrets.AZURE_APP_ID is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_PASSWORD" ]]; then
+            echo "AZURE_PASSWORD is not set. Please check if secrets.AZURE_PASSWORD is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_TENANT" ]]; then
+            echo "AZURE_TENANT is not set. Please check if secrets.AZURE_TENANT is in the repository."
+            exit 1
+          fi
+
+      - name: Checkout GitHub Repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+
+      - name: Install Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Azure CLI Login Using Service Principal
+        run: az login --service-principal --username "$AZURE_APP_ID" --password "$AZURE_PASSWORD" --tenant "$AZURE_TENANT"
+
+      - name: Run Azure-related integration test
+        run: go test -v ./pkg/driver/deployment/azure_functions_test.go

.github/workflows/e2e_azure.yaml

@@ -0,0 +1,68 @@
+name: End-to-End Azure Functions Tests
+
+on:
+  push:
+    branches:
+      - main # Trigger the workflow when code is pushed to the main branch
+  pull_request:
+    branches:
+      - main # Trigger the workflow when the PR targets the main branch
+  workflow_dispatch: # Allows manual triggering of the workflow
+
+env:
+  GOOS: linux
+  GO111MODULE: on
+
+jobs:
+  test-azure:
+    name: Test E2E Azure Functions Cloud Deployment
+    runs-on: ubuntu-24.04
+    env:
+      AZURE_APP_ID: ${{ secrets.AZURE_APP_ID }}
+      AZURE_PASSWORD: ${{ secrets.AZURE_PASSWORD }}
+      AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
+
+    steps:
+      - name: Check if environment variables are set  # Validate secrets are passed
+        run: |
+          if [[ -z "$AZURE_APP_ID" ]]; then
+            echo "AZURE_APP_ID is not set. Please check if secrets.AZURE_APP_ID is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_PASSWORD" ]]; then
+            echo "AZURE_PASSWORD is not set. Please check if secrets.AZURE_PASSWORD is in the repository."
+            exit 1
+          fi
+          if [[ -z "$AZURE_TENANT" ]]; then
+            echo "AZURE_TENANT is not set. Please check if secrets.AZURE_TENANT is in the repository."
+            exit 1
+          fi
+
+      - name: Checkout GitHub Repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version
+
+      - name: Install Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Azure CLI Login Using Service Principal
+        run: az login --service-principal --username "$AZURE_APP_ID" --password "$AZURE_PASSWORD" --tenant "$AZURE_TENANT"
+
+      - name: Build and Run Loader
+        run: go run cmd/loader.go --config cmd/config_azure_trace.json
+
+      - name: Check the output
+        run: test -f "data/out/experiment_duration_5.csv" && test $(grep true data/out/experiment_duration_5.csv | wc -l) -eq 0 # test the output file for errors (true means failure to invoke)

.gitignore

@@ -211,3 +211,9 @@ tools/plotter/test-out
 
 data/traces/azure_*
 data/traces/day*
+
+# Azure deployment
+azure_functions_for_zip/
+azurefunctions_setup/shared_azure_workload/exec_func.py
+azure_functions_for_zip_test/
+function*.zip

azurefunctions_setup/azurefunctionsconfig.yaml

@@ -0,0 +1,6 @@
+# azurefunctionsconfig.yaml
+azurefunctionsconfig:
+  resource_group: invitro-rg  # Name of the resource group
+  storage_account_name: invitrostorage  # Name of the storage account
+  function_app_name: invitro-functionapp  # Name of the function app
+  location: EastUS  # Region where resource created

azurefunctions_setup/host.json

@@ -0,0 +1,16 @@
+{
+    "version": "2.0",
+    "logging": {
+      "applicationInsights": {
+        "samplingSettings": {
+          "isEnabled": true,
+          "excludedTypes": "Request"
+        }
+      }
+    },
+    "extensionBundle": {
+      "id": "Microsoft.Azure.Functions.ExtensionBundle",
+      "version": "[4.*, 5.0.0)"
+    }
+
+}

azurefunctions_setup/local.settings.json

@@ -0,0 +1,8 @@
+{
+    "IsEncrypted": false,
+    "Values": {
+      "FUNCTIONS_WORKER_RUNTIME": "python",
+      "AzureWebJobsFeatureFlags": "EnableWorkerIndexing",
+      "AzureWebJobsStorage": ""
+    }
+}

azurefunctions_setup/requirements.txt

@@ -0,0 +1,3 @@
+azure-functions
+numpy>=1.21,<1.26
+psutil>=5.9,<6.0

azurefunctions_setup/shared_azure_workload/azurefunctionsworkload.py

@@ -0,0 +1,56 @@
+import time
+import socket
+import json
+import azure.functions as func
+import logging
+
+# Note: exec_func.py is not stored here permanently. It is copied from server/trace-func-py during setup.
+from .exec_func import execute_function
+
+# Global variable for hostname
+hostname = socket.gethostname()
+
+def main(req: func.HttpRequest) -> func.HttpResponse:
+    logging.info("Processing request.")
+
+    start_time = time.time()
+
+    # Parse JSON request body
+    try:
+        req_body = req.get_json()
+        logging.info(f"Request body: {req_body}")
+    except ValueError:
+        logging.error("Invalid JSON received.")
+        return func.HttpResponse(
+            json.dumps({"error": "Invalid JSON"}),
+            status_code=400,
+            mimetype="application/json"
+        )
+
+    runtime_milliseconds = req_body.get('RuntimeInMilliSec', 1000)
+    memory_mebibytes = req_body.get('MemoryInMebiBytes', 128)
+
+    logging.info(f"Runtime requested: {runtime_milliseconds} ms, Memory: {memory_mebibytes} MiB")
+
+    # Call the execute_function, which needs to be copied from server/trace-func-py/exec_func.py
+    duration = execute_function("",runtime_milliseconds,memory_mebibytes)
+    result_msg = f"Workload completed in {duration} microseconds"
+
+    # Prepare the response
+    response = {
+        "Status": "Success",
+        "Function": req.url.split("/")[-1],
+        "MachineName": hostname,
+        "ExecutionTime": int((time.time() - start_time) * 1_000_000),  # Total time (includes HTTP, workload, and response prep)
+        "DurationInMicroSec": duration,  # Time spent on the workload itself
+        "MemoryUsageInKb": memory_mebibytes * 1024,
+        "Message": result_msg
+    }
+
+    logging.info(f"Response: {response}")
+
+    return func.HttpResponse(
+        json.dumps(response),
+        status_code=200,
+        mimetype="application/json"
+    )

azurefunctions_setup/shared_azure_workload/function.json

@@ -0,0 +1,18 @@
+{
+    "bindings": [
+      {
+        "authLevel": "anonymous",
+        "type": "httpTrigger",
+        "direction": "in",
+        "name": "req",
+        "methods": ["post"]
+      },
+      {
+        "type": "http",
+        "direction": "out",
+        "name": "$return"
+      }
+    ],
+    "scriptFile": "azurefunctionsworkload.py"
+}
+

cmd/config_azure_trace.json

@@ -0,0 +1,28 @@
+{
+    "Seed": 42,
+
+    "Platform": "AzureFunctions",
+    "InvokeProtocol" : "http1", 
+    "EndpointPort": 80, 
+
+    "BusyLoopOnSandboxStartup": false,
+
+    "TracePath": "data/traces/example",
+    "Granularity": "minute",
+    "OutputPathPrefix": "data/out/experiment",
+    "IATDistribution": "exponential",
+    "CPULimit": "1vCPU",    
+    "ExperimentDuration": 5,
+    "WarmupDuration": 0,
+
+    "IsPartiallyPanic": false,
+    "EnableZipkinTracing": false,
+    "EnableMetricsScrapping": false,
+    "MetricScrapingPeriodSeconds": 15,
+    "AutoscalingMetric": "concurrency",
+
+    "GRPCConnectionTimeoutSeconds": 15,     
+    "GRPCFunctionTimeoutSeconds": 900,     
+
+    "DAGMode": false
+}

cmd/loader.go

@@ -95,6 +95,7 @@ func main() {
 		common.PlatformOpenWhisk,
 		common.PlatformAWSLambda,
 		common.PlatformDirigent,
+		common.PlatformAzureFunctions,
 	}
 	if !slices.Contains(supportedPlatforms, cfg.Platform) {
 		log.Fatal("Unsupported platform!")
@@ -149,7 +150,7 @@ func parseYAMLSpecification(cfg *config.LoaderConfiguration) string {
 	case "firecracker":
 		return "workloads/firecracker/trace_func_go.yaml"
 	default:
-		if cfg.Platform != common.PlatformDirigent {
+		if cfg.Platform != common.PlatformDirigent && cfg.Platform != common.PlatformAzureFunctions {
 			log.Fatal("Invalid 'YAMLSelector' parameter.")
 		}
 	}

docs/loader.md

@@ -284,4 +284,61 @@ Note:
   - Under `Manage Quota`, select `AWS Lambda` service and click `View quotas` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas))
   - Under `Quota name`, select `Concurrent executions` and click `Request increase at account level` (Alternatively, click [here](https://us-east-1.console.aws.amazon.com/servicequotas/home/services/lambda/quotas/L-B99A9384))
   - Under `Increase quota value`, input `1000` and click `Request`
-  - Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
+  - Await AWS Support Team to approve the request. The request may take several days or weeks to be approved.
+
+## Using Azure Functions
+
+**Pre-requisites:**
+1. Microsoft Azure account with an active subscription ID
+2. Existing Service Principal for authentication (refer to Notes section)
+3. Go installed
+4. Python3 installed
+
+**Quick Setup for Azure Deployment:**
+1. Install the Azure CLI and verify installation:
+    ```bash
+    curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+    az --version
+    ```
+2. Use existing Service Principal credentials in order login to Azure.
+    ```bash
+    az login --service-principal --username $AZURE_APP_ID --password $AZURE_PASSWORD --tenant $AZURE_TENANT
+    ```
+   > Refer to Note section for generation of Service Principal credentials
+3. Start the Azure Functions deployment experiment:
+    ```bash
+    go run cmd/loader.go --config cmd/config_azure_trace.json
+    ```
+---
+Notes:
+
+- A Service Principal must be created before running the experiment, as some environments (e.g., CloudLab nodes) do not support interactive login or browser-based authentication. You can create the required Service Principal using the Azure Portal.
+1. Login to [Azure Portal](https://portal.azure.com) and search for **"App registrations"**.
+2. Click **+ New registration**:
+   - Name: `InVitro`
+   - Supported account types: *Single tenant* (default)
+   - Skip Redirect URI
+   - Click **Register**
+3. Once registered, note down the following:
+   - **Application (client) ID** → This is your `AZURE_APP_ID`
+   - **Directory (tenant) ID** → This is your `AZURE_TENANT`
+4. Click **Add a certificate or secret**
+   - Click **+ New client secret**
+   - Add a description and choose an expiry (e.g., 6 months)
+   - Click **Add**
+   - Copy the secret **value** → This is your `AZURE_PASSWORD`
+5. Assign roles:
+   - Go to **Subscriptions** → Select your subscription
+   - Click **Access Control (IAM)** → **+ Add** → **Add role assignment**
+   - Choose **Contributor** under "Privileged administrator roles"
+   - Assign access to: *User, group, or service principal*
+   - Under "Members", search for your new app registration (`InVitro`)
+   - Click **Review + Assign**  
+6. Set the following environment variables in your experiment environment: 
+     ```bash
+     export AZURE_APP_ID=<appId>
+     export AZURE_PASSWORD=<password>
+     export AZURE_TENANT=<tenant>
+     ```
+- Current deployment is via ZIP
+- Python is used for deployment workload as Go is not supported in Consumption Plan

docs/multi_loader.md

@@ -31,7 +31,7 @@ More information regarding the metrics that can be collected at the end of each
 - **activator** – Captures Knative Activator logs which includes health and readiness status updates for service endpoints it manages
 - **autoscaler** – Tracks Knative autoscaler decisions, scaling events, and resource utilization
 - **top** – Provides CPU and memory usage statistics for each nodes
-- **prometheus** – Captures a snapshot of Prometheus’s TSDB, which includes system-wide performance metrics. The snapshot can be restored in an active Prometheus instance for further analysis. For details, see the *Restore Prometheus Data* section in [this guide](https://devopstales.github.io/home/backup-and-retore-prometheus/).
+- **prometheus** – Captures a snapshot of Prometheus’s TSDB, which includes system-wide performance metrics. The snapshot can be restored in an active Prometheus instance for further analysis. For details, see the *Restore Prometheus Data* section in [this guide](https://devopstales.github.io/kubernetes/backup-and-retore-prometheus/).
 
 
 ### LoaderStudy

pkg/common/constants.go

@@ -108,10 +108,11 @@ var ValidCPULimits = []string{CPULimit1vCPU, CPULimitGCP}
 
 // platform
 const (
-	PlatformKnative   string = "knative"
-	PlatformDirigent  string = "dirigent"
-	PlatformOpenWhisk string = "openwhisk"
-	PlatformAWSLambda string = "awslambda"
+	PlatformKnative        string = "knative"
+	PlatformDirigent       string = "dirigent"
+	PlatformOpenWhisk      string = "openwhisk"
+	PlatformAWSLambda      string = "awslambda"
+	PlatformAzureFunctions string = "azurefunctions"
 )
 
 // dirigent backend