Skip to content

Commit 7bcad4b

Browse files
kristan66kristan66
committed
Added ability to delete data
1 parent 32ff61d commit 7bcad4b

File tree

7 files changed

+420
-4
lines changed

7 files changed

+420
-4
lines changed

‎src/KeeperData.Bridge/Controllers/ImportController.cs‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
using KeeperData.Bridge.Worker.Tasks;
2+
using KeeperData.Core.Database;
3+
using KeeperData.Core.ETL.Abstract;
24
using KeeperData.Core.Reporting;
35
using KeeperData.Core.Reporting.Dtos;
46
using KeeperData.Infrastructure.Storage;
@@ -11,7 +13,8 @@ namespace KeeperData.Bridge.Controllers;
1113
public class ImportController(
1214
ITaskProcessBulkFiles taskProcessBulkFiles,
1315
IImportReportingService importReportingService,
14-
ILogger<ImportController> logger) : ControllerBase
16+
ILogger<ImportController> logger,
17+
ICollectionManagementService collectionManagementService) : ControllerBase
1518
{
1619
/// <summary>
1720
/// Starts a bulk file import process asynchronously.
@@ -239,4 +242,99 @@ public async Task<IActionResult> GetFileReports(Guid importId, CancellationToken
239242
});
240243
}
241244
}
245+
246+
/// <summary>
247+
/// Deletes a specific MongoDB collection by name.
248+
/// The collection name must be defined in DataSetDefinitions.
249+
/// </summary>
250+
/// <param name="collectionName">The name of the collection to delete</param>
251+
/// <param name="cancellationToken">Cancellation token</param>
252+
/// <returns>Success response with deleted collection name, or 404 if collection not found in definitions</returns>
253+
[HttpDelete("collections/{collectionName}")]
254+
[ProducesResponseType(typeof(DeleteCollectionResponse), StatusCodes.Status200OK)]
255+
[ProducesResponseType(typeof(ErrorResponse), StatusCodes.Status404NotFound)]
256+
[ProducesResponseType(typeof(ErrorResponse), StatusCodes.Status499ClientClosedRequest)]
257+
public async Task<IActionResult> DeleteCollection(string collectionName, CancellationToken cancellationToken = default)
258+
{
259+
logger.LogInformation("Received request to delete collection: {CollectionName}", collectionName);
260+
261+
var result = await collectionManagementService.DeleteCollectionAsync(collectionName, cancellationToken);
262+
263+
if (!result.Success)
264+
{
265+
if (result.Error is ArgumentException)
266+
{
267+
return NotFound(new ErrorResponse
268+
{
269+
Message = result.Message,
270+
Timestamp = DateTime.UtcNow
271+
});
272+
}
273+
274+
if (result.Error is OperationCanceledException)
275+
{
276+
return StatusCode(499, new ErrorResponse
277+
{
278+
Message = result.Message,
279+
Timestamp = DateTime.UtcNow
280+
});
281+
}
282+
283+
return StatusCode(500, new ErrorResponse
284+
{
285+
Message = result.Message,
286+
Timestamp = DateTime.UtcNow
287+
});
288+
}
289+
290+
return Ok(new DeleteCollectionResponse
291+
{
292+
CollectionName = result.CollectionName,
293+
Success = true,
294+
Message = result.Message,
295+
DeletedAtUtc = result.OperatedAtUtc
296+
});
297+
}
298+
299+
/// <summary>
300+
/// Deletes all MongoDB collections defined in DataSetDefinitions.
301+
/// </summary>
302+
/// <param name="cancellationToken">Cancellation token</param>
303+
/// <returns>Summary of deleted collections</returns>
304+
[HttpDelete("collections")]
305+
[ProducesResponseType(typeof(DeleteCollectionsResponse), StatusCodes.Status200OK)]
306+
[ProducesResponseType(typeof(ErrorResponse), StatusCodes.Status499ClientClosedRequest)]
307+
public async Task<IActionResult> DeleteAllCollections(CancellationToken cancellationToken = default)
308+
{
309+
logger.LogInformation("Received request to delete all collections");
310+
311+
var result = await collectionManagementService.DeleteAllCollectionsAsync(cancellationToken);
312+
313+
if (!result.Success)
314+
{
315+
if (result.Error is OperationCanceledException)
316+
{
317+
return StatusCode(499, new ErrorResponse
318+
{
319+
Message = result.Message,
320+
Timestamp = DateTime.UtcNow
321+
});
322+
}
323+
324+
return StatusCode(500, new ErrorResponse
325+
{
326+
Message = result.Message,
327+
Timestamp = DateTime.UtcNow
328+
});
329+
}
330+
331+
return Ok(new DeleteCollectionsResponse
332+
{
333+
DeletedCollections = result.DeletedCollections,
334+
TotalCount = result.TotalCount,
335+
Success = true,
336+
Message = result.Message,
337+
DeletedAtUtc = result.OperatedAtUtc
338+
});
339+
}
242340
}

‎src/KeeperData.Core/Database/ICollectionManagementService.cs‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
namespace KeeperData.Core.Database;
2+
3+
/// <summary>
4+
/// Service for managing MongoDB collections defined in DataSetDefinitions.
5+
/// </summary>
6+
public interface ICollectionManagementService
7+
{
8+
/// <summary>
9+
/// Deletes a specific MongoDB collection by name.
10+
/// The collection name must be defined in DataSetDefinitions.
11+
/// </summary>
12+
/// <param name="collectionName">The name of the collection to delete</param>
13+
/// <param name="cancellationToken">Cancellation token</param>
14+
/// <returns>Result containing success status and collection name</returns>
15+
Task<DeleteCollectionResult> DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default);
16+
17+
/// <summary>
18+
/// Deletes all MongoDB collections defined in DataSetDefinitions.
19+
/// </summary>
20+
/// <param name="cancellationToken">Cancellation token</param>
21+
/// <returns>Result containing list of deleted collections</returns>
22+
Task<DeleteAllCollectionsResult> DeleteAllCollectionsAsync(CancellationToken cancellationToken = default);
23+
}
24+
25+
/// <summary>
26+
/// Result of deleting a single collection.
27+
/// </summary>
28+
public record DeleteCollectionResult
29+
{
30+
/// <summary>
31+
/// Gets the name of the deleted collection.
32+
/// </summary>
33+
public required string CollectionName { get; init; }
34+
35+
/// <summary>
36+
/// Gets a value indicating whether the operation was successful.
37+
/// </summary>
38+
public required bool Success { get; init; }
39+
40+
/// <summary>
41+
/// Gets the result message.
42+
/// </summary>
43+
public required string Message { get; init; }
44+
45+
/// <summary>
46+
/// Gets the error exception if the operation failed, or null if successful.
47+
/// </summary>
48+
public Exception? Error { get; init; }
49+
50+
/// <summary>
51+
/// Gets the UTC timestamp of the operation.
52+
/// </summary>
53+
public DateTime OperatedAtUtc { get; init; }
54+
}
55+
56+
/// <summary>
57+
/// Result of deleting all collections.
58+
/// </summary>
59+
public record DeleteAllCollectionsResult
60+
{
61+
/// <summary>
62+
/// Gets the list of collection names that were deleted.
63+
/// </summary>
64+
public required IReadOnlyList<string> DeletedCollections { get; init; }
65+
66+
/// <summary>
67+
/// Gets the total number of collections deleted.
68+
/// </summary>
69+
public required int TotalCount { get; init; }
70+
71+
/// <summary>
72+
/// Gets a value indicating whether all collections were deleted successfully.
73+
/// </summary>
74+
public required bool Success { get; init; }
75+
76+
/// <summary>
77+
/// Gets the result message.
78+
/// </summary>
79+
public required string Message { get; init; }
80+
81+
/// <summary>
82+
/// Gets the error exception if the operation failed, or null if successful.
83+
/// </summary>
84+
public Exception? Error { get; init; }
85+
86+
/// <summary>
87+
/// Gets the UTC timestamp of the operation.
88+
/// </summary>
89+
public DateTime OperatedAtUtc { get; init; }
90+
}

‎src/KeeperData.Core/Database/Impl/CollectionManagementService.cs‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
using KeeperData.Core.ETL.Abstract;
2+
using Microsoft.Extensions.Logging;
3+
using Microsoft.Extensions.Options;
4+
using MongoDB.Driver;
5+
6+
namespace KeeperData.Core.Database.Impl;
7+
8+
/// <summary>
9+
/// Implementation of collection management service for MongoDB operations.
10+
/// </summary>
11+
public class CollectionManagementService : ICollectionManagementService
12+
{
13+
private readonly IMongoClient _mongoClient;
14+
private readonly IDatabaseConfig _databaseConfig;
15+
private readonly IDataSetDefinitions _dataSetDefinitions;
16+
private readonly ILogger<CollectionManagementService> _logger;
17+
18+
public CollectionManagementService(
19+
IMongoClient mongoClient,
20+
IOptions<IDatabaseConfig> databaseConfig,
21+
IDataSetDefinitions dataSetDefinitions,
22+
ILogger<CollectionManagementService> logger)
23+
{
24+
_mongoClient = mongoClient ?? throw new ArgumentNullException(nameof(mongoClient));
25+
_databaseConfig = databaseConfig?.Value ?? throw new ArgumentNullException(nameof(databaseConfig));
26+
_dataSetDefinitions = dataSetDefinitions ?? throw new ArgumentNullException(nameof(dataSetDefinitions));
27+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
28+
}
29+
30+
public async Task<DeleteCollectionResult> DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default)
31+
{
32+
_logger.LogInformation("Attempting to delete collection: {CollectionName}", collectionName);
33+
34+
try
35+
{
36+
// Validate that the collection is defined in DataSetDefinitions
37+
var collectionDefined = _dataSetDefinitions.All.Any(d => d.Name.Equals(collectionName, StringComparison.OrdinalIgnoreCase));
38+
39+
if (!collectionDefined)
40+
{
41+
var message = $"Collection '{collectionName}' is not defined in DataSetDefinitions.";
42+
_logger.LogWarning("{Message}", message);
43+
return new DeleteCollectionResult
44+
{
45+
CollectionName = collectionName,
46+
Success = false,
47+
Message = message,
48+
Error = new ArgumentException(message),
49+
OperatedAtUtc = DateTime.UtcNow
50+
};
51+
}
52+
53+
var database = _mongoClient.GetDatabase(_databaseConfig.DatabaseName);
54+
await database.DropCollectionAsync(collectionName, cancellationToken);
55+
56+
_logger.LogInformation("Successfully deleted collection: {CollectionName}", collectionName);
57+
58+
return new DeleteCollectionResult
59+
{
60+
CollectionName = collectionName,
61+
Success = true,
62+
Message = $"Collection '{collectionName}' deleted successfully.",
63+
Error = null,
64+
OperatedAtUtc = DateTime.UtcNow
65+
};
66+
}
67+
catch (OperationCanceledException ex)
68+
{
69+
var message = $"Delete collection operation was cancelled for collection: {collectionName}";
70+
_logger.LogWarning(ex, "{Message}", message);
71+
return new DeleteCollectionResult
72+
{
73+
CollectionName = collectionName,
74+
Success = false,
75+
Message = message,
76+
Error = ex,
77+
OperatedAtUtc = DateTime.UtcNow
78+
};
79+
}
80+
catch (Exception ex)
81+
{
82+
var message = $"Failed to delete collection '{collectionName}': {ex.Message}";
83+
_logger.LogError(ex, "{Message}", message);
84+
return new DeleteCollectionResult
85+
{
86+
CollectionName = collectionName,
87+
Success = false,
88+
Message = message,
89+
Error = ex,
90+
OperatedAtUtc = DateTime.UtcNow
91+
};
92+
}
93+
}
94+
95+
public async Task<DeleteAllCollectionsResult> DeleteAllCollectionsAsync(CancellationToken cancellationToken = default)
96+
{
97+
_logger.LogInformation("Attempting to delete all collections");
98+
99+
try
100+
{
101+
var database = _mongoClient.GetDatabase(_databaseConfig.DatabaseName);
102+
var collectionsToDelete = _dataSetDefinitions.All.Select(d => d.Name).ToList();
103+
var deletedCollections = new List<string>();
104+
105+
foreach (var collectionName in collectionsToDelete)
106+
{
107+
try
108+
{
109+
await database.DropCollectionAsync(collectionName, cancellationToken);
110+
deletedCollections.Add(collectionName);
111+
_logger.LogInformation("Deleted collection: {CollectionName}", collectionName);
112+
}
113+
catch (MongoCommandException ex) when (ex.Code == 26) // Namespace not found (collection doesn't exist)
114+
{
115+
_logger.LogWarning("Collection {CollectionName} does not exist, skipping", collectionName);
116+
// Continue with next collection
117+
}
118+
catch (Exception ex)
119+
{
120+
_logger.LogError(ex, "Failed to delete collection {CollectionName}", collectionName);
121+
// Continue with next collection
122+
}
123+
}
124+
125+
_logger.LogInformation("Successfully deleted {Count} collection(s)", deletedCollections.Count);
126+
127+
return new DeleteAllCollectionsResult
128+
{
129+
DeletedCollections = deletedCollections,
130+
TotalCount = deletedCollections.Count,
131+
Success = true,
132+
Message = $"{deletedCollections.Count} collection(s) deleted successfully.",
133+
Error = null,
134+
OperatedAtUtc = DateTime.UtcNow
135+
};
136+
}
137+
catch (OperationCanceledException ex)
138+
{
139+
var message = "Delete all collections operation was cancelled";
140+
_logger.LogWarning(ex, "{Message}", message);
141+
return new DeleteAllCollectionsResult
142+
{
143+
DeletedCollections = new List<string>(),
144+
TotalCount = 0,
145+
Success = false,
146+
Message = message,
147+
Error = ex,
148+
OperatedAtUtc = DateTime.UtcNow
149+
};
150+
}
151+
catch (Exception ex)
152+
{
153+
var message = $"Failed to delete collections: {ex.Message}";
154+
_logger.LogError(ex, "{Message}", message);
155+
return new DeleteAllCollectionsResult
156+
{
157+
DeletedCollections = new List<string>(),
158+
TotalCount = 0,
159+
Success = false,
160+
Message = message,
161+
Error = ex,
162+
OperatedAtUtc = DateTime.UtcNow
163+
};
164+
}
165+
}
166+
}

‎src/KeeperData.Core/ETL/Impl/IngestionPipeline.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ private async Task<FileIngestionMetrics> ProcessCsvRecordsAsync(
491491
}
492492

493493
// Check progress and report every ProgressUpdateInterval records (default: 10)
494-
if (metrics.RecordsProcessed > lastReportedRecordCount &&
494+
if (metrics.RecordsProcessed > lastReportedRecordCount &&
495495
metrics.RecordsProcessed - lastReportedRecordCount >= ProgressUpdateInterval)
496496
{
497497
LogProgressIfNeeded(metrics.RecordsProcessed, fileKey);

0 commit comments

Comments
 (0)