-
Notifications
You must be signed in to change notification settings - Fork 52
Export History job #494
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
YunchuWang
wants to merge
12
commits into
main
Choose a base branch
from
wangbill/exportfinal
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Export History job #494
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit
Hold shift + click to select a range
871eefe
export job
YunchuWang c0b946b
Merge branch 'main' into wangbill/exportfinal
YunchuWang 13c0dcb
get history events
YunchuWang 52bf21c
v1
YunchuWang 30fcebc
Merge branch 'main' into wangbill/exportfinal
YunchuWang ced2c4b
revert
YunchuWang 5749aeb
Merge branch 'wangbill/exportfinal' of https://github.com/microsoft/d…
YunchuWang 46b01e4
warning fix
YunchuWang 2df6b53
unit tests
YunchuWang 208aade
feedback
YunchuWang 00ee88d
Merge branch 'main' into wangbill/exportfinal
YunchuWang b45b151
Merge branch 'main' into wangbill/exportfinal
YunchuWang File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <Nullable>enable</Nullable> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles> | ||
| <CompilerGeneratedFilesOutputPath>$(BaseIntermediateOutputPath)Generated</CompilerGeneratedFilesOutputPath> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Azure.Identity" /> | ||
| <PackageReference Include="Grpc.Net.Client" /> | ||
| <PackageReference Include="Microsoft.DurableTask.Generators" OutputItemType="Analyzer" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\src\Client\AzureManaged\Client.AzureManaged.csproj" /> | ||
| <ProjectReference Include="..\..\src\Worker\AzureManaged\Worker.AzureManaged.csproj" /> | ||
| <ProjectReference Include="..\..\src\ExportHistory\ExportHistory.csproj" /> | ||
| </ItemGroup> | ||
| </Project> | ||
|
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| ### Variables | ||
| @baseUrl = http://localhost:5010 | ||
| @jobId = export-job-12345 | ||
|
|
||
| ### Create a new batch export job | ||
| # @name createBatchExportJob | ||
| POST {{baseUrl}}/export-jobs | ||
| Content-Type: application/json | ||
|
|
||
| { | ||
| "jobId": "{{jobId}}", | ||
| "mode": "Batch", | ||
| "completedTimeFrom": "2025-10-01T00:00:00Z", | ||
| "completedTimeTo": "2025-11-06T23:59:59Z", | ||
| "container": "export-history", | ||
| # "prefix": "exports/", | ||
| "maxInstancesPerBatch": 1, | ||
| "runtimeStatus": [] | ||
| } | ||
|
|
||
| ### Create a new continuous export job | ||
| # @name createContinuousExportJob | ||
| POST {{baseUrl}}/export-jobs | ||
| Content-Type: application/json | ||
|
|
||
| { | ||
| "jobId": "export-job-continuous-123", | ||
| "mode": "Continuous", | ||
| "container": "export-history", | ||
| # "prefix": "continuous-exports/", | ||
| "maxInstancesPerBatch": 1000 | ||
| # "runtimeStatus": ["asdasd"] | ||
| } | ||
|
|
||
| ### Create an export job with default storage (no container specified) | ||
| # @name createExportJobWithDefaultStorage | ||
| POST {{baseUrl}}/export-jobs | ||
| Content-Type: application/json | ||
| { | ||
| "jobId": "export-job-default-storage", | ||
| "mode": "Batch", | ||
| "completedTimeFrom": "2024-01-01T00:00:00Z", | ||
| "completedTimeTo": "2024-12-31T23:59:59Z", | ||
| "maxInstancesPerBatch": 100 | ||
| } | ||
|
|
||
| ### Get a specific export job by ID | ||
| # Note: This endpoint can be used to verify the export job was created and check its status | ||
| # The ID in the URL should match the jobId used in create request | ||
| GET {{baseUrl}}/export-jobs/{{jobId}} | ||
|
|
||
| ### List all export jobs | ||
| GET {{baseUrl}}/export-jobs/list | ||
|
|
||
| ### List export jobs with filters | ||
| ### Filter by status | ||
| GET {{baseUrl}}/export-jobs/list?status=Active | ||
|
|
||
| ### Filter by job ID prefix | ||
| GET {{baseUrl}}/export-jobs/list?jobIdPrefix=export-job- | ||
|
|
||
| ### Filter by creation time range | ||
| GET {{baseUrl}}/export-jobs/list?createdFrom=2024-01-01T00:00:00Z&createdTo=2024-12-31T23:59:59Z | ||
|
|
||
| ### Combined filters | ||
| GET {{baseUrl}}/export-jobs/list?status=Completed&jobIdPrefix=export-job-&pageSize=50 | ||
|
|
||
| ### Delete an export job | ||
| # DELETE {{baseUrl}}/export-jobs/{{jobId}} | ||
|
|
||
| # Delete a continuous export job | ||
| DELETE {{baseUrl}}/export-jobs/export-job-continuous-123jk | ||
|
|
||
| ### Tips: | ||
| # - Replace the baseUrl variable if your application runs on a different port | ||
| # - The jobId variable can be changed to test different export job instances | ||
| # - Export modes: | ||
| # - "Batch": Exports all instances within a time range (requires completedTimeTo) | ||
| # - "Continuous": Continuously exports instances from a start time (completedTimeTo must be null) | ||
| # - Runtime status filters (valid values): | ||
| # - "Completed": Exports only completed orchestrations | ||
| # - "Failed": Exports only failed orchestrations | ||
| # - "Terminated": Exports only terminated orchestrations | ||
| # - "ContinuedAsNew": Exports only continued-as-new orchestrations | ||
| # - Dates are in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ) | ||
| # - You can use the REST Client extension in VS Code to execute these requests | ||
| # - The @name directive allows referencing the response in subsequent requests | ||
| # - Export jobs run asynchronously; use GET to check the status after creation | ||
|
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using Microsoft.AspNetCore.Mvc; | ||
| using Microsoft.DurableTask; | ||
| using Microsoft.DurableTask.Client; | ||
| using Microsoft.DurableTask.ExportHistory; | ||
| using ExportHistoryWebApp.Models; | ||
|
|
||
| namespace ExportHistoryWebApp.Controllers; | ||
|
|
||
| /// <summary> | ||
| /// Controller for managing export history jobs through a REST API. | ||
| /// Provides endpoints for creating, reading, listing, and deleting export jobs. | ||
| /// </summary> | ||
| [ApiController] | ||
| [Route("export-jobs")] | ||
| public class ExportJobController : ControllerBase | ||
| { | ||
| readonly ExportHistoryClient exportHistoryClient; | ||
| readonly ILogger<ExportJobController> logger; | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="ExportJobController"/> class. | ||
| /// </summary> | ||
| /// <param name="exportHistoryClient">Client for managing export history jobs.</param> | ||
| /// <param name="logger">Logger for recording controller operations.</param> | ||
| public ExportJobController( | ||
| ExportHistoryClient exportHistoryClient, | ||
| ILogger<ExportJobController> logger) | ||
| { | ||
| this.exportHistoryClient = exportHistoryClient ?? throw new ArgumentNullException(nameof(exportHistoryClient)); | ||
| this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Creates a new export job based on the provided configuration. | ||
| /// </summary> | ||
| /// <param name="request">The export job creation request.</param> | ||
| /// <returns>The created export job description.</returns> | ||
| [HttpPost] | ||
| public async Task<ActionResult<ExportJobDescription>> CreateExportJob([FromBody] CreateExportJobRequest request) | ||
| { | ||
| if (request == null) | ||
| { | ||
| return this.BadRequest("createExportJobRequest cannot be null"); | ||
| } | ||
|
|
||
| try | ||
| { | ||
| ExportDestination? destination = null; | ||
| if (!string.IsNullOrEmpty(request.Container)) | ||
| { | ||
| destination = new ExportDestination(request.Container) | ||
| { | ||
| Prefix = request.Prefix, | ||
| }; | ||
| } | ||
|
|
||
| ExportJobCreationOptions creationOptions = new ExportJobCreationOptions( | ||
| mode: request.Mode, | ||
| completedTimeFrom: request.CompletedTimeFrom, | ||
| completedTimeTo: request.CompletedTimeTo, | ||
| destination: destination, | ||
| jobId: request.JobId, | ||
| format: request.Format, | ||
| runtimeStatus: request.RuntimeStatus, | ||
| maxInstancesPerBatch: request.MaxInstancesPerBatch); | ||
|
|
||
| ExportHistoryJobClient jobClient = await this.exportHistoryClient.CreateJobAsync(creationOptions); | ||
| ExportJobDescription description = await jobClient.DescribeAsync(); | ||
|
|
||
| this.logger.LogInformation("Created new export job with ID: {JobId}", description.JobId); | ||
|
|
||
| return this.CreatedAtAction(nameof(GetExportJob), new { id = description.JobId }, description); | ||
| } | ||
| catch (ArgumentException ex) | ||
| { | ||
| this.logger.LogError(ex, "Validation failed while creating export job {JobId}", request.JobId); | ||
| return this.BadRequest(ex.Message); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| this.logger.LogError(ex, "Error creating export job {JobId}", request.JobId); | ||
YunchuWang marked this conversation as resolved.
Dismissed
Show dismissed
Hide dismissed
|
||
| return this.StatusCode(500, "An error occurred while creating the export job"); | ||
| } | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| /// <summary> | ||
| /// Retrieves a specific export job by its ID. | ||
| /// </summary> | ||
| /// <param name="id">The ID of the export job to retrieve.</param> | ||
| /// <returns>The export job description if found.</returns> | ||
| [HttpGet("{id}")] | ||
| public async Task<ActionResult<ExportJobDescription>> GetExportJob(string id) | ||
| { | ||
| try | ||
| { | ||
| ExportJobDescription? job = await this.exportHistoryClient.GetJobAsync(id); | ||
| return this.Ok(job); | ||
| } | ||
| catch (ExportJobNotFoundException) | ||
| { | ||
| return this.NotFound(); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| this.logger.LogError(ex, "Error retrieving export job {JobId}", id); | ||
YunchuWang marked this conversation as resolved.
Dismissed
Show dismissed
Hide dismissed
|
||
| return this.StatusCode(500, "An error occurred while retrieving the export job"); | ||
| } | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| /// <summary> | ||
| /// Lists all export jobs, optionally filtered by query parameters. | ||
| /// </summary> | ||
| /// <param name="status">Optional filter by job status.</param> | ||
| /// <param name="jobIdPrefix">Optional filter by job ID prefix.</param> | ||
| /// <param name="createdFrom">Optional filter for jobs created after this time.</param> | ||
| /// <param name="createdTo">Optional filter for jobs created before this time.</param> | ||
| /// <param name="pageSize">Optional page size for pagination.</param> | ||
| /// <param name="continuationToken">Optional continuation token for pagination.</param> | ||
| /// <returns>A collection of export job descriptions.</returns> | ||
| [HttpGet("list")] | ||
| public async Task<ActionResult<IEnumerable<ExportJobDescription>>> ListExportJobs( | ||
| [FromQuery] ExportJobStatus? status = null, | ||
| [FromQuery] string? jobIdPrefix = null, | ||
| [FromQuery] DateTimeOffset? createdFrom = null, | ||
| [FromQuery] DateTimeOffset? createdTo = null, | ||
| [FromQuery] int? pageSize = null, | ||
| [FromQuery] string? continuationToken = null) | ||
| { | ||
| this.logger.LogInformation("GET list endpoint called with method: {Method}", this.HttpContext.Request.Method); | ||
YunchuWang marked this conversation as resolved.
Dismissed
Show dismissed
Hide dismissed
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| try | ||
| { | ||
| ExportJobQuery? query = null; | ||
| if ( | ||
| status.HasValue || | ||
| !string.IsNullOrEmpty(jobIdPrefix) || | ||
| createdFrom.HasValue || | ||
| createdTo.HasValue || | ||
| pageSize.HasValue || | ||
| !string.IsNullOrEmpty(continuationToken) | ||
| ) | ||
| { | ||
| query = new ExportJobQuery | ||
| { | ||
| Status = status, | ||
| JobIdPrefix = jobIdPrefix, | ||
| CreatedFrom = createdFrom, | ||
| CreatedTo = createdTo, | ||
| PageSize = pageSize, | ||
| ContinuationToken = continuationToken, | ||
| }; | ||
| } | ||
|
|
||
| AsyncPageable<ExportJobDescription> jobs = this.exportHistoryClient.ListJobsAsync(query); | ||
|
|
||
| // Collect all jobs from the async pageable | ||
| List<ExportJobDescription> jobList = new List<ExportJobDescription>(); | ||
| await foreach (ExportJobDescription job in jobs) | ||
| { | ||
| jobList.Add(job); | ||
| } | ||
|
|
||
| return this.Ok(jobList); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| this.logger.LogError(ex, "Error retrieving export jobs"); | ||
| return this.StatusCode(500, "An error occurred while retrieving export jobs"); | ||
| } | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| /// <summary> | ||
| /// Deletes an export job by its ID. | ||
| /// </summary> | ||
| /// <param name="id">The ID of the export job to delete.</param> | ||
| /// <returns>No content if successful.</returns> | ||
| [HttpDelete("{id}")] | ||
| public async Task<IActionResult> DeleteExportJob(string id) | ||
| { | ||
| this.logger.LogInformation("DELETE endpoint called for job ID: {JobId}", id); | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| try | ||
| { | ||
| ExportHistoryJobClient jobClient = this.exportHistoryClient.GetJobClient(id); | ||
| await jobClient.DeleteAsync(); | ||
| this.logger.LogInformation("Successfully deleted export job {JobId}", id); | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return this.NoContent(); | ||
| } | ||
| catch (ExportJobNotFoundException) | ||
| { | ||
| this.logger.LogWarning("Export job {JobId} not found for deletion", id); | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return this.NotFound(); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| this.logger.LogError(ex, "Error deleting export job {JobId}", id); | ||
YunchuWang marked this conversation as resolved.
Dismissed
Show dismissed
Hide dismissed
|
||
| return this.StatusCode(500, "An error occurred while deleting the export job"); | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
YunchuWang marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
| } | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.