Skip to content

Commit 86d0104

Browse files
nadomaninadomani
authored
Merge pull request #224 from microsoft/dev/nbarzilai/sharde_package_topublic
Sample updates (May 2025)
2 parents 1547a25 + df3d69f commit 86d0104

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1441
-343
lines changed

‎Backend/src/Contracts/ItemJobMetadata.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33
// </copyright>
44

55
using System;
6-
using Fabric_Extension_BE_Boilerplate.Contracts.FabricAPI.Workload;
76

87
namespace Boilerplate.Contracts {
98
public class ItemJobMetadata {
109
public string JobType { get; set; }
1110
public Guid JobInstanceId { get; set; }
12-
public JobInstanceStatus Status { get; set; }
1311
public object ErrorDetails { get; set; }
1412
public DateTime? CanceledTime { get; set; }
13+
public bool UseOneLake { get; set; }
14+
public bool IsCanceled => CanceledTime != null;
1515
}
1616
}

‎Backend/src/Contracts/LakehouseFile.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// <copyright company="Microsoft">
2+
// Copyright (c) Microsoft. All rights reserved.
3+
// </copyright>
4+
5+
namespace Boilerplate.Contracts
6+
{
7+
public class LakehouseFile
8+
{
9+
public string Name { get; init; }
10+
11+
public string Path { get; init; }
12+
13+
public bool IsDirectory { get; init; }
14+
15+
}
16+
}

‎Backend/src/Contracts/OneLakePathContainer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ internal class OneLakePathData
1111
public string Name { get; init; }
1212
public bool IsShortcut { get; init; }
1313
public string AccountType { get; init; }
14+
public bool IsDirectory { get; init; }
1415
}
1516

1617
internal class OneLakePathContainer

‎Backend/src/Controllers/EventhouseController.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ namespace Boilerplate.Controllers
1414
public class EventhouseController : ControllerBase
1515
{
1616
private static readonly IList<string> EventhubFabricScopes = new[] { $"{EnvironmentConstants.FabricBackendResourceId}/Eventhouse.Read.All" };
17-
1817
private readonly ILogger<EventhouseController> _logger;
1918
private readonly IHttpContextAccessor _httpContextAccessor;
2019
private readonly IAuthenticationService _authenticationService;
@@ -33,11 +32,20 @@ IHttpClientService httpClientService
3332
_httpClientService = httpClientService;
3433
}
3534

35+
/// <summary>
36+
/// Retrieves an Eventhouse by its ID within a specified workspace.
37+
/// In order for this method to succeed, the caller must have the appropriate permissions to access the Eventhouse.
38+
/// Permissions and scopes are defined in Entra ID application. For this request the scope is: FabricEventhouse.Read.All
39+
/// For more information please follow the link:
40+
/// https://learn.microsoft.com/en-us/fabric/workload-development-kit/fabric-data-plane#api-permissions
41+
/// </summary>
42+
/// <param name="workspaceId">The ID of the workspace containing the Eventhouse.</param>
43+
/// <param name="eventhouseId">The ID of the Eventhouse to retrieve.</param>
44+
/// <returns>An IActionResult containing the Eventhouse details if successful, or an error message if the request fails.</returns>
3645
[HttpGet("eventhouse/{workspaceId}/{eventhouseId}")]
3746
public async Task<IActionResult> GetEventhouse(Guid workspaceId, Guid eventhouseId)
3847
{
3948
_logger.LogInformation("GetEventhouse: get eventhouse '{0}' in workspace '{1}'", eventhouseId, workspaceId);
40-
4149
var authorizationContext = await _authenticationService.AuthenticateDataPlaneCall(_httpContextAccessor.HttpContext, allowedScopes: new string[] {WorkloadScopes.FabricEventhouseReadAll});
4250
var token = await _authenticationService.GetAccessTokenOnBehalfOf(authorizationContext, EventhubFabricScopes);
4351

@@ -51,7 +59,6 @@ public async Task<IActionResult> GetEventhouse(Guid workspaceId, Guid eventhouse
5159
}
5260
var eventhouse = await response.Content.ReadAsAsync<EventhouseItem>();
5361
return Ok(eventhouse);
54-
5562
}
5663
}
5764
}

‎Backend/src/Controllers/KqlDatabaseController.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,20 @@ public KqlDatabaseController(
3535
_httpClientService = new HttpClient();
3636
}
3737

38+
/// <summary>
39+
/// Executes a KQL query against a specified database.
40+
/// In order for this method to succeed, the caller must have the appropriate permissions to access the KQL data base.
41+
/// Permissions and scopes are defined in Entra ID application. For this request the scope is: KQLDatabase.ReadWrite.All
42+
/// For more information please follow the link:
43+
/// https://learn.microsoft.com/en-us/fabric/workload-development-kit/fabric-data-plane#api-permissions
44+
/// </summary>
45+
/// <param name="request">The request containing the query details.</param>
46+
/// <returns>An IActionResult containing the query results or an error message.</returns>
47+
/// <remarks>
48+
/// This endpoint authenticates the user, constructs an HTTP request to the KQL service, and processes the response.
49+
/// If the query is successful, the results are returned as a DataTable.
50+
/// If the query fails, an appropriate error message is returned.
51+
/// </remarks>
3852
[HttpPost("KqlDatabases/query")]
3953
public async Task<IActionResult> QueryKqlDatabase([FromBody] QueryKqlDatabaseRequest request)
4054
{

‎Backend/src/Controllers/LakeHouseController.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,15 @@ public async Task<IActionResult> GetTablesAsync(Guid workspaceId, Guid lakehouse
9393
return Ok(tables);
9494
}
9595

96+
[HttpGet("onelake/{workspaceId:guid}/{lakehouseId:guid}/files")]
97+
public async Task<IActionResult> GetFilesAsync(Guid workspaceId, Guid lakehouseId)
98+
{
99+
var authorizationContext = await _authenticationService.AuthenticateDataPlaneCall(_httpContextAccessor.HttpContext, allowedScopes: ScopesForReadLakehouseFile);
100+
var token = await _authenticationService.GetAccessTokenOnBehalfOf(authorizationContext, OneLakeConstants.OneLakeScopes);
101+
var files = await _lakeHouseClientService.GetLakehouseFiles(token, workspaceId, lakehouseId);
102+
return Ok(files);
103+
}
104+
96105

97106
}
98107
}

‎Backend/src/Items/Item1.cs‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
using Microsoft.Extensions.Logging;
1414
using System;
1515
using System.Collections.Generic;
16+
using System.IO;
1617
using System.Linq;
1718
using System.Threading.Tasks;
1819

@@ -81,6 +82,14 @@ public override async Task ExecuteJob(string jobType, Guid jobInstanceId, JobInv
8182
return;
8283
}
8384

85+
var jobMetadata = new ItemJobMetadata
86+
{
87+
JobInstanceId = jobInstanceId,
88+
JobType = jobType,
89+
UseOneLake = _metadata.UseOneLake
90+
};
91+
await ItemMetadataStore.UpsertJob(TenantObjectId, ItemObjectId, jobInstanceId, jobMetadata);
92+
8493
var token = await AuthenticationService.GetAccessTokenOnBehalfOf(AuthorizationContext, OneLakeConstants.OneLakeScopes);
8594

8695
var op1 = _metadata.Operand1;
@@ -95,9 +104,21 @@ public override async Task ExecuteJob(string jobType, Guid jobInstanceId, JobInv
95104
await Task.Delay(TimeSpan.FromSeconds(60 * 8));
96105
}
97106

107+
try
108+
{
109+
// Reload job metadata to check later if the job was cancelled
110+
jobMetadata = await ItemMetadataStore.LoadJob(TenantObjectId, ItemObjectId, jobInstanceId);
111+
}
112+
catch (FileNotFoundException exc)
113+
{
114+
// Demonstrating a way to recover job metadata if it has been deleted.
115+
Logger.LogWarning(exc, $"{nameof(ExecuteJob)} - Recreating missing job {jobInstanceId} metadata in tenant {TenantObjectId} item {ItemObjectId}.");
116+
await ItemMetadataStore.UpsertJob(TenantObjectId, ItemObjectId, jobInstanceId, jobMetadata);
117+
}
118+
98119
// Write result to Lakehouse if job is not cancelled
99-
if (!ItemMetadataStore.JobCancelRequestExists(TenantObjectId, ItemObjectId, jobInstanceId)) {
100-
var filePath = GetCalculationResultFilePath(jobType, jobInstanceId);
120+
if (!jobMetadata.IsCanceled) {
121+
var filePath = GetCalculationResultFilePath(jobMetadata);
101122
await OneLakeClientService.WriteToOneLakeFile(token, filePath, result);
102123

103124
_metadata.LastCalculationResultLocation = filePath;
@@ -115,36 +136,43 @@ public override async Task<ItemJobInstanceState> GetJobState(string jobType, Gui
115136
};
116137
}
117138

118-
var token = await AuthenticationService.GetAccessTokenOnBehalfOf(AuthorizationContext, OneLakeConstants.OneLakeScopes);
119-
120-
var filePath = GetCalculationResultFilePath(jobType, jobInstanceId);
121-
var fileExists = await OneLakeClientService.CheckIfFileExists(token, filePath);
139+
if (!ItemMetadataStore.ExistsJob(TenantObjectId, ItemObjectId, jobInstanceId))
140+
{
141+
Logger.LogError($"{nameof(GetJobState)} - Job {jobInstanceId} metadata does not exist in tenant {TenantObjectId} item {ItemObjectId}.");
142+
return new ItemJobInstanceState() { Status = JobInstanceStatus.Failed };
143+
}
122144

123-
if (ItemMetadataStore.JobCancelRequestExists(TenantObjectId, ItemObjectId, jobInstanceId))
145+
var jobMetadata = await ItemMetadataStore.LoadJob(TenantObjectId, ItemObjectId, jobInstanceId);
146+
if (jobMetadata.IsCanceled)
124147
{
125148
return new ItemJobInstanceState { Status = JobInstanceStatus.Cancelled };
126149
}
127150

151+
var filePath = GetCalculationResultFilePath(jobMetadata);
152+
var token = await AuthenticationService.GetAccessTokenOnBehalfOf(AuthorizationContext, OneLakeConstants.OneLakeScopes);
153+
var fileExists = await OneLakeClientService.CheckIfFileExists(token, filePath);
154+
128155
return new ItemJobInstanceState
129156
{
130157
Status = fileExists ? JobInstanceStatus.Completed : JobInstanceStatus.InProgress,
131158
};
132159
}
133160

134-
private string GetCalculationResultFilePath(string jobType, Guid jobInstanceId)
161+
private string GetCalculationResultFilePath(ItemJobMetadata jobMetadata)
135162
{
163+
var jobInstanceId = jobMetadata.JobInstanceId;
136164
var typeToFileName = new Dictionary<string, string>
137165
{
138166
{ Item1JobType.ScheduledJob, $"CalculationResult_{jobInstanceId}.txt" },
139167
{ Item1JobType.CalculateAsText, $"CalculationResult_{jobInstanceId}.txt" },
140168
{ Item1JobType.LongRunningCalculateAsText, $"CalculationResult_{jobInstanceId}.txt" },
141169
{ Item1JobType.CalculateAsParquet, $"CalculationResult_{jobInstanceId}.parquet" }
142170
};
143-
typeToFileName.TryGetValue(jobType, out var fileName);
171+
typeToFileName.TryGetValue(jobMetadata.JobType, out var fileName);
144172

145173
if (fileName != null)
146174
{
147-
return _metadata.UseOneLake
175+
return jobMetadata.UseOneLake
148176
? OneLakeClientService.GetOneLakeFilePath(WorkspaceObjectId,ItemObjectId, fileName)
149177
: OneLakeClientService.GetOneLakeFilePath(_metadata.Lakehouse.WorkspaceId, _metadata.Lakehouse.Id, fileName);
150178
}

‎Backend/src/Items/ItemBase.cs‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,6 @@ public abstract class ItemBase<TItem, TItemMetadata, TItemClientMetadata> : IIte
2424
where TItem : ItemBase<TItem, TItemMetadata, TItemClientMetadata>
2525
where TItemMetadata : class
2626
{
27-
protected static readonly JsonSerializerOptions ClientSerializationOptions = new JsonSerializerOptions
28-
{
29-
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
30-
Converters = { new JsonStringEnumConverter() },
31-
};
32-
3327
public Guid TenantObjectId { get; private set; }
3428

3529
public Guid WorkspaceObjectId { get; private set; }
@@ -148,15 +142,30 @@ public async Task Delete()
148142
public abstract Task<ItemJobInstanceState> GetJobState(string jobType, Guid jobInstanceId);
149143

150144
public async Task CancelJob(string jobType, Guid jobInstanceId) {
151-
var jobMetadata = new ItemJobMetadata
145+
ItemJobMetadata jobMetadata;
146+
if (!ItemMetadataStore.ExistsJob(TenantObjectId, ItemObjectId, jobInstanceId))
152147
{
153-
JobType = jobType,
154-
JobInstanceId = jobInstanceId,
155-
Status = JobInstanceStatus.Cancelled,
156-
ErrorDetails = null,
157-
CanceledTime = DateTime.UtcNow,
158-
};
159-
await ItemMetadataStore.UpsertJobCancel(TenantObjectId, ItemObjectId, jobType, jobInstanceId, jobMetadata);
148+
// Demonstrating a way to handle removed or missing job metadata
149+
Logger.LogWarning($"{nameof(CancelJob)} - Recreating missing job {jobInstanceId} metadata in tenant {TenantObjectId} item {ItemObjectId}.");
150+
jobMetadata = new ItemJobMetadata
151+
{
152+
JobType = jobType,
153+
JobInstanceId = jobInstanceId
154+
};
155+
}
156+
else
157+
{
158+
jobMetadata = await ItemMetadataStore.LoadJob(TenantObjectId, ItemObjectId, jobInstanceId);
159+
}
160+
161+
if (jobMetadata.IsCanceled)
162+
{
163+
return;
164+
}
165+
166+
jobMetadata.CanceledTime = DateTime.UtcNow;
167+
168+
await ItemMetadataStore.UpsertJob(TenantObjectId, ItemObjectId, jobInstanceId, jobMetadata);
160169
}
161170

162171
protected async Task SaveChanges()

‎Backend/src/Services/AuthenticationService.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -397,8 +397,11 @@ private void ValidateAnyScope(IEnumerable<Claim> claims, IList<string> allowedSc
397397

398398
if (!scopes.Any(s => allowedScopes.Contains(s)))
399399
{
400-
_logger.LogError("Missing or invalid 'scp' claim");
401-
throw new AuthenticationException("Invalid scopes");
400+
var invalidScopesMessage = "Workload's Entra ID application is missing required scopes";
401+
402+
_logger.LogError(invalidScopesMessage);
403+
throw new AuthenticationException(invalidScopesMessage)
404+
.WithDetail(ErrorCodes.Authentication.AuthError, invalidScopesMessage, [("title", "Missing or invalid scopes")]);
402405
}
403406
}
404407

‎Backend/src/Services/IItemMetadataStore.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ namespace Boilerplate.Services
1111
public interface IItemMetadataStore
1212
{
1313
Task Upsert<TItemMetadata>(Guid tenantObjectId, Guid itemObjectId, CommonItemMetadata commonMetadata, TItemMetadata typeSpecificMetadata);
14-
15-
Task UpsertJobCancel(Guid tenantObjectId, Guid itemObjectId, string jobType, Guid jobInstanceId, ItemJobMetadata itemJobMetadata);
1614
Task<ItemMetadata<TItemMetadata>> Load<TItemMetadata>(Guid tenantObjectId, Guid itemObjectId);
17-
1815
Task Delete(Guid tenantObjectId, Guid itemObjectId);
1916
bool Exists(Guid tenantObjectId, Guid itemObjectId);
20-
bool JobCancelRequestExists(Guid tenantObjectId, Guid itemObjectId, Guid jobInstanceId);
17+
18+
Task UpsertJob(Guid tenantObjectId, Guid itemObjectId, Guid jobInstanceId, ItemJobMetadata itemJobMetadata);
19+
Task<ItemJobMetadata> LoadJob(Guid tenantObjectId, Guid itemObjectId, Guid jobInstanceId);
20+
bool ExistsJob(Guid tenantObjectId, Guid itemObjectId, Guid jobInstanceId);
2121
}
2222
}

0 commit comments

Comments
 (0)