tests: Use pytest as a test runner for Jupyter and Python samples

.github/workflows/pre-commit.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - run: pip install -r dev-requirements.txt
       - name: Run Pre-Commit
         run: pre-commit run --all-files

.github/workflows/run-samples.yml

@@ -0,0 +1,86 @@
+name: Run Samples
+on:
+  # By design pull_request_target event run against the version of the workflow in the target branch.
+  # So you have to merge changes to this workflow to observe the effects.
+  pull_request_target:
+    branches:
+      - main
+    paths:
+      - scenarios/**
+      - .infra/deployments/**/*.bicep
+jobs:
+  check-if-external:
+    runs-on: ubuntu-latest
+    outputs:
+      environment: ${{ steps.set-environment.outputs.result }}
+    steps:
+      - uses: actions/github-script@v7
+        id: set-environment
+        with:
+          script: |
+            const actionInitiator = context.payload.sender.login;
+            const org = "Azure-Samples";
+            let isPublicMember = true;
+
+            // Check if initiator is a public member of the org
+            try {
+              await github.rest.orgs.checkPublicMembershipForUser({
+                org,
+                username: actionInitiator
+              });
+            } catch (error) {
+              if (error.status != 404) {
+                throw new Error("Unknown error", {cause: error});
+              }
+
+              console.debug([
+                `User is not a public member of the organization "${org}"`,
+                "",
+                `If you are a Microsoft employee, you can join the "${org}" org and set your org membership visibility to public: https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/publicizing-or-hiding-organization-membership#changing-the-visibility-of-your-organization-membership`
+              ].join("\n"));
+
+              isPublicMember = false;
+            }
+
+
+            const isPullRequestEvent = ["pull_request", "pull_request_target"].includes(context.eventName);
+
+            if (!(isPublicMember && isPullRequestEvent)) {
+              return "external-contribution";
+            }
+            return "";
+          result-encoding: string
+  run-samples:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    needs: check-if-external
+    runs-on: ubuntu-latest
+    # Require manual approval if initiator is not a public member of Azure-Samples
+    environment: ${{ needs.check-if-external.outputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - uses: actions/setup-python@v5
+        with:
+            python-version: "3.9"
+      - name: Install dev dependencies
+        run: |
+            pip install -r dev-requirements.txt
+      - uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      - name: Deploy resources
+        run: |
+            principalId="$(az ad sp show --id ${{ secrets.AZURE_CLIENT_ID }} -o tsv --query id)"
+            az deployment sub create --location eastus \
+              --template-file .infra/deployment/main.bicep \
+              --parameters principalType=ServicePrincipal \
+              --parameters principalId="$principalId" \
+              -o json > deployment.json
+      - name: Run samples
+        run:
+            pytest --changed-samples-only-from ${{ github.base_ref }}

.gitignore

@@ -1,4 +1,20 @@
 
+# Ignore output of infra deployment used for testing
+deployment.json
+
+.virtual_documents/
+
+# Ignore output files generated by running samples
+*.output.ipynb
+*output.json
+*output.jsonl
+*outputs.json
+*outputs.jsonl
+*result.json
+*result.jsonl
+*results.json
+*results.jsonl
+
 ## Python Gitignore
 ## From: https://github.com/github/gitignore/blob/main/Python.gitignore
 

.infra/deployment/main.bicep

@@ -0,0 +1,110 @@
+targetScope = 'subscription'
+param workspaceName string = 'azureai_samples_hub'
+param projectName string = 'azureai_samples_proj'
+param resourceGroupName string = 'rg-azureai-samples-validation-${utcNow('yyyyMM')}'
+param location string = 'eastus2'
+@description('The ID of the principal (user, service principal, etc...) to create role assignments for.')
+param principalId string = ''
+
+@description('The Type of the principal (user, service principal, etc...) to create role assignments for.')
+@allowed([ 'User', 'ServicePrincipal', '' ])
+param principalType string = ''
+
+var acsName = 'acs-samples-${uniqueString(rg.id, workspaceName, workspaceName)}'
+
+resource rg 'Microsoft.Resources/resourceGroups@2023-07-01' = {
+  #disable-next-line use-stable-resource-identifiers
+  name: resourceGroupName
+  location: location
+}
+
+module acs 'modules/acs.bicep' = { name: 'acs', params: { name: acsName, location: location }, scope: rg }
+
+module workspace_hub 'modules/workspace_hub.bicep' = {
+  name: 'workspace-hub'
+  params: {
+    location: location
+    name: workspaceName
+    searchName: acs.outputs.name
+
+  }
+  scope: rg
+}
+
+module project 'modules/ai_project.bicep' = {
+  name: 'project'
+  params: {
+    location: location
+    workspaceHubID: workspace_hub.outputs.id
+    name: projectName
+  }
+  scope: rg
+}
+
+var deployments = {
+  gpt4: {
+    name: 'gpt-4'
+    properties: {
+      model: {
+        format: 'OpenAI'
+        name: 'gpt-4'
+        version: '1106-Preview'
+      }
+    }
+  }
+
+  text_embedding_ada_002: {
+    name: 'text-embedding-ada-002'
+    properties: {
+      model: {
+        format: 'OpenAI'
+        name: 'text-embedding-ada-002'
+        version: '2'
+      }
+      raiPolicyName: 'Microsoft.Default'
+      versionUpgradeOption: 'OnceNewDefaultVersionAvailable'
+      type: 'Azure.OpenAI'
+      sku: {
+        name: 'Standard'
+        capacity: 1
+      }
+    }
+  }
+}
+
+var roleDefinitionIds = [
+  'a001fd3d-188f-4b5d-821b-7da978bf7442' // Cognitive Services OpenAI Contributor
+  'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor
+]
+
+module role_assignments 'modules/role_assignment.bicep' = [for rd in roleDefinitionIds: if (!empty(principalId)) {
+  name: 'role_assignment-${rd}'
+  params: {
+    principalId: principalId
+    principalType: principalType
+    roleDefinitionId: rd
+  }
+  scope: rg
+}]
+
+@batchSize(1)
+module project_deployments 'modules/ai_project_deployment.bicep' = [for deployment in items(deployments): {
+  name: 'project_deployment-${deployment.value.name}'
+  params: {
+    name: deployment.value.name
+    properties: deployment.value.properties
+    ai_services_name: workspace_hub.outputs.ai_services_name
+  }
+  scope: rg
+}]
+
+@description('The ID of the subscription deployed to.')
+output subscription_id string = subscription().subscriptionId
+@description('The name of the resource group deployed to.')
+output resource_group_name string = rg.name
+@description('The name of the Azure AI Project.')
+output project_name string = project.outputs.name
+output project_location string = project.outputs.location
+output azure_openai_endpoint string = workspace_hub.outputs.azure_openai_endpoint
+output azure_openai_gpt4_api_version string = '2024-08-01-preview'
+output azure_openai_gpt4_deployment_name string = deployments.gpt4.name

.infra/deployment/modules/acs.bicep

@@ -0,0 +1,58 @@
+@description('Service name must only contain lowercase letters, digits or dashes, cannot use dash as the first two or last one characters, cannot contain consecutive dashes, and is limited between 2 and 60 characters in length.')
+@minLength(2)
+@maxLength(60)
+param name string
+
+@allowed([
+  'free'
+  'basic'
+  'standard'
+  'standard2'
+  'standard3'
+  'storage_optimized_l1'
+  'storage_optimized_l2'
+])
+@description('The pricing tier of the search service you want to create (for example, basic or standard).')
+param sku string = 'standard'
+
+@description('Replicas distribute search workloads across the service. You need at least two replicas to support high availability of query workloads (not applicable to the free tier).')
+@minValue(1)
+@maxValue(12)
+param replicaCount int = 1
+
+@description('Partitions allow for scaling of document count as well as faster indexing by sharding your index over multiple search units.')
+@allowed([
+  1
+  2
+  3
+  4
+  6
+  12
+])
+param partitionCount int = 1
+
+@description('Applicable only for SKUs set to standard3. You can set this property to enable a single, high density partition that allows up to 1000 indexes, which is much higher than the maximum indexes allowed for any other SKU.')
+@allowed([
+  'default'
+  'highDensity'
+])
+param hostingMode string = 'default'
+
+@description('Location for all resources.')
+param location string = resourceGroup().location
+
+resource search 'Microsoft.Search/searchServices@2020-08-01' = {
+  name: name
+  location: location
+  sku: {
+    name: sku
+  }
+  properties: {
+    replicaCount: replicaCount
+    partitionCount: partitionCount
+    hostingMode: hostingMode
+  }
+}
+
+output name string = search.name
+output id string = search.id

.infra/deployment/modules/ai_project.bicep

@@ -0,0 +1,22 @@
+param name string
+param nameFriendly string = name
+param workspaceHubID string
+param location string = resourceGroup().location
+
+resource project 'Microsoft.MachineLearningServices/workspaces@2023-10-01' = {
+  name: name
+  #disable-next-line BCP187
+  kind: 'Project'
+  location: location
+  identity: {
+    type: 'SystemAssigned'
+  }
+  properties: {
+    friendlyName: nameFriendly
+    #disable-next-line BCP037
+    hubResourceId: workspaceHubID
+  }
+}
+
+output name string = project.name
+output location string = project.location

.infra/deployment/modules/ai_project_deployment.bicep

@@ -0,0 +1,32 @@
+param name string
+param properties object
+param ai_services_name string
+
+var defaults = {
+  raiPolicyName: 'Microsoft.Default'
+  versionUpgradeOption: 'OnceNewDefaultVersionAvailable'
+  type: 'Azure.OpenAI'
+  sku: {
+    name: 'Standard'
+    capacity: 20
+  }
+}
+
+var properties_with_defaults = union(defaults, properties)
+
+#disable-next-line BCP081
+resource aiResource 'Microsoft.CognitiveServices/accounts@2024-06-01-preview' existing = {
+  name: ai_services_name
+
+}
+
+#disable-next-line BCP081
+resource deployment 'Microsoft.CognitiveServices/accounts/deployments@2024-06-01-preview' = {
+  parent: aiResource
+  name: name
+  sku: properties_with_defaults.sku
+  properties: {
+    model: properties_with_defaults.model
+    versionUpgradeOption: properties_with_defaults.versionUpgradeOption
+  }
+}

.infra/deployment/modules/container_registry.bicep

@@ -0,0 +1,15 @@
+param name string
+param location string = resourceGroup().id
+
+resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-preview' = {
+  name: name
+  location: location
+  sku: {
+    name: 'Standard'
+  }
+  properties: {
+    adminUserEnabled: false
+  }
+}
+
+output id string = containerRegistry.id

.infra/deployment/modules/keyvault.bicep

@@ -0,0 +1,19 @@
+param name string
+param location string = resourceGroup().location
+param tenantId string = tenant().tenantId
+
+resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = {
+  name: name
+  location: location
+  properties: {
+    tenantId: tenantId
+    sku: {
+      name: 'standard'
+      family: 'A'
+    }
+    accessPolicies: []
+    enableSoftDelete: true
+  }
+}
+
+output id string = keyVault.id