Add support for tokenless authentication for GitHub Actions configured with OpenID Connect with Azure User Managed Identity (or Service Principal) (#385)

## Changes
See
https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers
and
https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure

Technically, it should also work with Azure DevOps Workload Identity
Federation, once we figure out the environment variables:
https://techcommunity.microsoft.com/t5/azure-devops-blog/introduction-to-azure-devops-workload-identity-federation-oidc/ba-p/3908687

## Tests

setup:
```
resource "github_actions_environment_secret" "scope" {
  for_each    = github_repository_environment.default
  repository  = each.value.repository
  environment = each.value.environment
  secret_name = "ARM_CLIENT_ID"
  # this value is not a secret as well
  plaintext_value = data.azurerm_user_assigned_identity.scopes[local.project_scopes[each.key]].client_id
}

...

resource "azurerm_federated_identity_credential" "oidc" {
  for_each            = github_repository_environment.default
  name                = "${local.org}-${each.value.repository}-${each.value.environment}-oidc"
  resource_group_name = local.resource_group_name
  audience            = ["api://AzureADTokenExchange"]
  issuer              = "https://token.actions.githubusercontent.com"
  parent_id           = data.azurerm_user_assigned_identity.scopes[local.project_scopes[each.key]].id
  subject             = "repo:${local.org}/${each.value.repository}:environment:${each.value.environment}"
}

...

resource "azurerm_user_assigned_identity" "scopes" {
  for_each = local.scopes
  name     = "labs-${each.key}-identity"

  resource_group_name = azurerm_resource_group.this.name
  location            = azurerm_resource_group.this.location
  // ...
}

...

resource "databricks_service_principal" "scopes" {
  provider = databricks.account
  for_each = local.scopes

  application_id = azurerm_user_assigned_identity.scopes[each.key].client_id
  display_name   = azurerm_user_assigned_identity.scopes[each.key].name
  external_id    = azurerm_user_assigned_identity.scopes[each.key].principal_id
}
```

result

<img width="603"
alt="_Experiment__Call_integration_tests_via_OIDC_·_databrickslabs_ucx_5a94b24"
src="https://github.com/databricks/databricks-sdk-py/assets/259697/33b08224-ceed-4c15-bfcd-32e0b58f8483">

- [ ] `make test` run locally
- [x] `make fmt` applied
- [ ] relevant integration tests applied

Author: nfx

databricks/sdk/core.py

@@ -217,6 +217,53 @@ def refreshed_headers() -> Dict[str, str]:
     return refreshed_headers
 
 
+@credentials_provider('github-oidc-azure', ['host', 'azure_client_id'])
+def github_oidc_azure(cfg: 'Config') -> Optional[HeaderFactory]:
+    if 'ACTIONS_ID_TOKEN_REQUEST_TOKEN' not in os.environ:
+        # not in GitHub actions
+        return None
+
+    # Client ID is the minimal thing we need, as otherwise we get AADSTS700016: Application with
+    # identifier 'https://token.actions.githubusercontent.com' was not found in the directory '...'.
+    if not cfg.is_azure:
+        return None
+
+    # See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers
+    headers = {'Authorization': f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"}
+    endpoint = f"{os.environ['ACTIONS_ID_TOKEN_REQUEST_URL']}&audience=api://AzureADTokenExchange"
+    response = requests.get(endpoint, headers=headers)
+    if not response.ok:
+        return None
+
+    # get the ID Token with aud=api://AzureADTokenExchange sub=repo:org/repo:environment:name
+    response_json = response.json()
+    if 'value' not in response_json:
+        return None
+
+    logger.info("Configured AAD token for GitHub Actions OIDC (%s)", cfg.azure_client_id)
+    params = {
+        'client_assertion_type': 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer',
+        'resource': cfg.effective_azure_login_app_id,
+        'client_assertion': response_json['value'],
+    }
+    aad_endpoint = cfg.arm_environment.active_directory_endpoint
+    if not cfg.azure_tenant_id:
+        # detect Azure AD Tenant ID if it's not specified directly
+        token_endpoint = cfg.oidc_endpoints.token_endpoint
+        cfg.azure_tenant_id = token_endpoint.replace(aad_endpoint, '').split('/')[0]
+    inner = ClientCredentials(client_id=cfg.azure_client_id,
+                              client_secret="", # we have no (rotatable) secrets in OIDC flow
+                              token_url=f"{aad_endpoint}{cfg.azure_tenant_id}/oauth2/token",
+                              endpoint_params=params,
+                              use_params=True)
+
+    def refreshed_headers() -> Dict[str, str]:
+        token = inner.token()
+        return {'Authorization': f'{token.token_type} {token.access_token}'}
+
+    return refreshed_headers
+
+
 class CliTokenSource(Refreshable):
 
     def __init__(self, cmd: List[str], token_type_field: str, access_token_field: str, expiry_field: str):
@@ -462,7 +509,7 @@ def auth_type(self) -> str:
     def __call__(self, cfg: 'Config') -> HeaderFactory:
         auth_providers = [
             pat_auth, basic_auth, metadata_service, oauth_service_principal, azure_service_principal,
-            azure_cli, external_browser, databricks_cli, runtime_native_auth
+            github_oidc_azure, azure_cli, external_browser, databricks_cli, runtime_native_auth
         ]
         for provider in auth_providers:
             auth_type = provider.auth_type()

tests/test_core.py

@@ -13,6 +13,7 @@
 import requests
 
 from databricks.sdk import WorkspaceClient
+from databricks.sdk.azure import ENVIRONMENTS, AzureEnvironment
 from databricks.sdk.core import (ApiClient, Config, CredentialsProvider,
                                  DatabricksCliTokenSource, DatabricksError,
                                  HeaderFactory, StreamingResponse,
@@ -510,3 +511,39 @@ def inner(h: BaseHTTPRequestHandler):
         assert 'foo' in res
 
     assert len(requests) == 2
+
+
+def test_github_oidc_flow_works_with_azure(monkeypatch):
+
+    def inner(h: BaseHTTPRequestHandler):
+        if 'audience=api://AzureADTokenExchange' in h.path:
+            auth = h.headers['Authorization']
+            assert 'Bearer gh-actions-token' == auth
+            h.send_response(200)
+            h.end_headers()
+            h.wfile.write(b'{"value": "this_is_jwt_token"}')
+            return
+        if '/oidc/oauth2/v2.0/authorize' == h.path:
+            h.send_response(301)
+            h.send_header('Location', f'http://{h.headers["Host"]}/mocked-tenant-id/irrelevant/part')
+            h.end_headers()
+            return
+        if '/mocked-tenant-id/oauth2/token' == h.path:
+            h.send_response(200)
+            h.end_headers()
+            h.wfile.write(b'{"expires_in": 100, "access_token": "this-is-it", "token_type": "Taker"}')
+
+    with http_fixture_server(inner) as host:
+        monkeypatch.setenv('ACTIONS_ID_TOKEN_REQUEST_URL', f'{host}/oidc')
+        monkeypatch.setenv('ACTIONS_ID_TOKEN_REQUEST_TOKEN', 'gh-actions-token')
+        ENVIRONMENTS[host] = AzureEnvironment(name=host,
+                                              service_management_endpoint=host + '/',
+                                              resource_manager_endpoint=host + '/',
+                                              active_directory_endpoint=host + '/')
+        cfg = Config(host=host,
+                     azure_workspace_resource_id=...,
+                     azure_client_id='test',
+                     azure_environment=host)
+        headers = cfg.authenticate()
+
+        assert {'Authorization': 'Taker this-is-it'} == headers