From 49d1e911a293ccd0429829e63220c75267f5201a Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 11:56:34 +0100 Subject: [PATCH 01/23] feat(auth): create ownership check middleware - Introduces a new `ownershipCheckMiddleware` to centralize the logic for verifying if a user owns a specific data item. - The middleware is configuration-driven, using `ModelConfig` to determine if a check is necessary for the requested action. - If a check is required, it fetches the item and provides it to the downstream handler, preventing redundant database reads. - Throws a `ForbiddenException` if the ownership check fails. - Adds a `FetchedItem` wrapper class for type-safe context provision. --- .../ownership_check_middleware.dart | 103 ++++++++++++++++++ pubspec.yaml | 4 + 2 files changed, 107 insertions(+) create mode 100644 lib/src/middlewares/ownership_check_middleware.dart diff --git a/lib/src/middlewares/ownership_check_middleware.dart b/lib/src/middlewares/ownership_check_middleware.dart new file mode 100644 index 0000000..58410a4 --- /dev/null +++ b/lib/src/middlewares/ownership_check_middleware.dart @@ -0,0 +1,103 @@ +import 'package:dart_frog/dart_frog.dart'; +import 'package:ht_api/src/rbac/permission_service.dart'; +import 'package:ht_api/src/registry/model_registry.dart'; +import 'package:ht_data_repository/ht_data_repository.dart'; +import 'package:ht_shared/ht_shared.dart'; + +/// A wrapper class to provide a fetched item into the request context. +/// +/// This ensures type safety and avoids providing a raw `dynamic` object, +/// which could lead to ambiguity if other dynamic objects are in the context. +class FetchedItem { + /// Creates a wrapper for the fetched item. + const FetchedItem(this.data); + + /// The fetched item data. + final T data; +} + +/// Middleware to check if the authenticated user is the owner of the requested +/// item. +/// +/// This middleware is designed to run on item-specific routes (e.g., `/[id]`). +/// It performs the following steps: +/// +/// 1. Determines if an ownership check is required for the current action +/// (GET, PUT, DELETE) based on the `ModelConfig`. +/// 2. If a check is required and the user is not an admin, it fetches the +/// item from the database. +/// 3. It then compares the item's owner ID with the authenticated user's ID. +/// 4. If the check fails, it throws a [ForbiddenException]. +/// 5. If the check passes, it provides the fetched item into the request +/// context via `context.provide>`. This prevents the +/// downstream route handler from needing to fetch the item again. +Middleware ownershipCheckMiddleware() { + return (handler) { + return (context) async { + final modelName = context.read(); + final modelConfig = context.read>(); + final user = context.read(); + final permissionService = context.read(); + final method = context.request.method; + final id = context.request.uri.pathSegments.last; + + ModelActionPermission permission; + switch (method) { + case HttpMethod.get: + permission = modelConfig.getItemPermission; + case HttpMethod.put: + permission = modelConfig.putPermission; + case HttpMethod.delete: + permission = modelConfig.deletePermission; + default: + // For other methods, no ownership check is performed here. + return handler(context); + } + + // If no ownership check is required or if the user is an admin, + // proceed to the next handler without fetching the item. + if (!permission.requiresOwnershipCheck || + permissionService.isAdmin(user)) { + return handler(context); + } + + if (modelConfig.getOwnerId == null) { + throw const OperationFailedException( + 'Internal Server Error: Model configuration error for ownership check.', + ); + } + + final userIdForRepoCall = user.id; + dynamic item; + + switch (modelName) { + case 'user': + final repo = context.read>(); + item = await repo.read(id: id, userId: userIdForRepoCall); + case 'user_app_settings': + final repo = context.read>(); + item = await repo.read(id: id, userId: userIdForRepoCall); + case 'user_content_preferences': + final repo = context.read>(); + item = await repo.read(id: id, userId: userIdForRepoCall); + default: + throw OperationFailedException( + 'Ownership check not implemented for model "$modelName".', + ); + } + + final itemOwnerId = modelConfig.getOwnerId!(item); + if (itemOwnerId != user.id) { + throw const ForbiddenException( + 'You do not have permission to access this item.', + ); + } + + final updatedContext = context.provide>( + () => FetchedItem(item), + ); + + return handler(updatedContext); + }; + }; +} diff --git a/pubspec.yaml b/pubspec.yaml index 56fade7..2339931 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -13,6 +13,9 @@ dependencies: ht_data_client: git: url: https://github.com/headlines-toolkit/ht-data-client.git + ht_data_mongodb: + git: + url: https://github.com/headlines-toolkit/ht-data-mongodb.git ht_data_postgres: git: url: https://github.com/headlines-toolkit/ht-data-postgres.git @@ -37,6 +40,7 @@ dependencies: logging: ^1.3.0 meta: ^1.16.0 + mongo_dart: ^0.10.5 postgres: ^3.5.6 shelf_cors_headers: ^0.1.5 uuid: ^4.5.1 From ea709e95020b852b73679d3ff647f949a8edb5de Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 12:01:16 +0100 Subject: [PATCH 02/23] feat(auth): apply ownership check middleware to item routes - Creates a new `_middleware.dart` file for the `/[id]` path. - Applies the `ownershipCheckMiddleware` to all item-specific requests (GET, PUT, DELETE). - This ensures ownership is verified after authentication and authorization checks have passed but before the route handler is executed. --- routes/api/v1/data/[id]/_middleware.dart | 19 +++++++++++++++++++ .../v1/data/{[id].dart => [id]/index.dart} | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 routes/api/v1/data/[id]/_middleware.dart rename routes/api/v1/data/{[id].dart => [id]/index.dart} (99%) diff --git a/routes/api/v1/data/[id]/_middleware.dart b/routes/api/v1/data/[id]/_middleware.dart new file mode 100644 index 0000000..3f1c5c2 --- /dev/null +++ b/routes/api/v1/data/[id]/_middleware.dart @@ -0,0 +1,19 @@ +import 'package:dart_frog/dart_frog.dart'; +import 'package:ht_api/src/middlewares/ownership_check_middleware.dart'; + +/// Middleware specific to the item-level `/api/v1/data/[id]` route path. +/// +/// This middleware applies the [ownershipCheckMiddleware] to perform an +/// ownership check on the requested item *after* the parent middleware +/// (`/api/v1/data/_middleware.dart`) has already performed authentication and +/// authorization checks. +/// +/// This ensures that only authorized users can proceed, and then this +/// middleware adds the final layer of security by verifying item ownership +/// for non-admin users when required by the model's configuration. +Handler middleware(Handler handler) { + // The `ownershipCheckMiddleware` will run after the middleware from + // `/api/v1/data/_middleware.dart` (authn, authz, model validation). + return handler.use(ownershipCheckMiddleware()); +} + diff --git a/routes/api/v1/data/[id].dart b/routes/api/v1/data/[id]/index.dart similarity index 99% rename from routes/api/v1/data/[id].dart rename to routes/api/v1/data/[id]/index.dart index 19ba614..e0cd998 100644 --- a/routes/api/v1/data/[id].dart +++ b/routes/api/v1/data/[id]/index.dart @@ -8,7 +8,7 @@ import 'package:ht_api/src/services/user_preference_limit_service.dart'; // Impo import 'package:ht_data_repository/ht_data_repository.dart'; import 'package:ht_shared/ht_shared.dart'; -import '../../../_middleware.dart'; // Assuming RequestId is here +import '../../../../_middleware.dart'; // Assuming RequestId is here /// Handles requests for the /api/v1/data/[id] endpoint. /// Dispatches requests to specific handlers based on the HTTP method. From cc9d32f9f2ec6a0586bb5f86f569320499027642 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 12:09:23 +0100 Subject: [PATCH 03/23] refactor(api): adapt GET /data to use native document query - Rewrites the `_handleGet` method in the collection route handler to accept a single, JSON-encoded `filter` parameter. - Removes the complex logic for translating individual URL parameters into a query map. - Eliminates the `_camelToSnake` helper, as the API now passes native model field names to the data layer. - This change simplifies the API handler and aligns it with a document-native query model, making it more flexible and robust. --- routes/api/v1/data/index.dart | 594 +++++++--------------------------- 1 file changed, 110 insertions(+), 484 deletions(-) diff --git a/routes/api/v1/data/index.dart b/routes/api/v1/data/index.dart index f5959aa..a4e4d73 100644 --- a/routes/api/v1/data/index.dart +++ b/routes/api/v1/data/index.dart @@ -1,610 +1,236 @@ +import 'dart:convert'; import 'dart:io'; import 'package:dart_frog/dart_frog.dart'; -import 'package:ht_api/src/rbac/permission_service.dart'; // Import PermissionService import 'package:ht_api/src/registry/model_registry.dart'; +import 'package:ht_api/src/services/user_preference_limit_service.dart'; import 'package:ht_data_repository/ht_data_repository.dart'; import 'package:ht_shared/ht_shared.dart'; -import '../../../_middleware.dart'; // Assuming RequestId is here - -/// Converts a camelCase string to snake_case. -String _camelToSnake(String input) { - return input - .replaceAllMapped( - RegExp('(? '_${match.group(0)}', - ) - .toLowerCase(); -} +import '../../../_middleware.dart'; // For RequestId /// Handles requests for the /api/v1/data collection endpoint. /// Dispatches requests to specific handlers based on the HTTP method. Future onRequest(RequestContext context) async { - // Read dependencies provided by middleware - final modelName = context.read(); - final modelConfig = context.read>(); - final requestId = context.read().id; - // User is guaranteed non-null by requireAuthentication() middleware - final authenticatedUser = context.read(); - final permissionService = context - .read(); // Read PermissionService - - // The main try/catch block here is removed to let the errorHandler middleware - // handle all exceptions thrown by the handlers below. switch (context.request.method) { case HttpMethod.get: - return _handleGet( - context, - modelName, - modelConfig, - authenticatedUser, - permissionService, // Pass PermissionService - requestId, - ); + return _handleGet(context); case HttpMethod.post: - return _handlePost( - context, - modelName, - modelConfig, - authenticatedUser, - permissionService, // Pass PermissionService - requestId, - ); - // Add cases for other methods if needed in the future + return _handlePost(context); default: - // Methods not allowed on the collection endpoint return Response(statusCode: HttpStatus.methodNotAllowed); } } -// --- GET Handler --- -/// Handles GET requests: Retrieves all items for the specified model. -/// -/// This handler implements model-specific filtering rules: -/// - **Headlines (`model=headline`):** -/// - Filterable by `q` (text query on title only). If `q` is present, -/// `topics` and `sources` are ignored. -/// Example: `/api/v1/data?model=headline&q=Dart+Frog` -/// - OR by a combination of: -/// - `topics` (comma-separated topic IDs). -/// Example: `/api/v1/data?model=headline&topics=topicId1,topicId2` -/// - `sources` (comma-separated source IDs). -/// Example: `/api/v1/data?model=headline&sources=sourceId1` -/// - Both `topics` and `sources` can be used together (AND logic). -/// Example: `/api/v1/data?model=headline&topics=topicId1&sources=sourceId1` -/// - Other parameters for headlines (e.g., `countries`) will result in a 400 Bad Request. +/// Handles GET requests: Retrieves a collection of items. /// -/// - **Sources (`model=source`):** -/// - Filterable by `q` (text query on name only). If `q` is present, -/// `countries`, `sourceTypes`, `languages` are ignored. -/// Example: `/api/v1/data?model=source&q=Tech+News` -/// - OR by a combination of: -/// - `countries` (comma-separated country ISO codes for `source.headquarters.iso_code`). -/// Example: `/api/v1/data?model=source&countries=US,GB` -/// - `sourceTypes` (comma-separated `SourceType` enum string values for `source.sourceType`). -/// Example: `/api/v1/data?model=source&sourceTypes=blog,news_agency` -/// - `languages` (comma-separated language codes for `source.language`). -/// Example: `/api/v1/data?model=source&languages=en,fr` -/// - These specific filters are ANDed if multiple are provided. -/// - Other parameters for sources will result in a 400 Bad Request. -/// -/// - **Topics (`model=topic`):** -/// - Filterable ONLY by `q` (text query on name only). -/// Example: `/api/v1/data?model=topic&q=Technology` -/// - Other parameters for topics will result in a 400 Bad Request. -/// -/// - **Countries (`model=country`):** -/// - Filterable ONLY by `q` (text query on name and isoCode). -/// Example: `/api/v1/data?model=country&q=United` (searches name and isoCode) -/// Example: `/api/v1/data?model=country&q=US` (searches name and isoCode) -/// - Other parameters for countries will result in a 400 Bad Request. -/// -/// - **Other Models (User, UserAppSettings, UserContentPreferences, RemoteConfig):** -/// - Currently support exact match for top-level query parameters passed directly. -/// - No specific complex filtering logic (like `_in` or `_contains`) is applied -/// by this handler for these models yet. The `HtDataInMemoryClient` can -/// process such queries if the `specificQueryForClient` map is constructed -/// with the appropriate keys by this handler in the future. -/// -/// Includes request metadata in the response. -Future _handleGet( - RequestContext context, - String modelName, - ModelConfig modelConfig, - User authenticatedUser, - PermissionService permissionService, - String requestId, -) async { - final queryParams = context.request.uri.queryParameters; - final startAfterId = queryParams['startAfterId']; - final limitParam = queryParams['limit']; - final sortByParam = queryParams['sortBy']; - final sortOrderRaw = queryParams['sortOrder']?.toLowerCase(); - final limit = limitParam != null ? int.tryParse(limitParam) : null; +/// This handler now accepts a single, JSON-encoded `filter` parameter for +/// MongoDB-style queries, along with `sort` and pagination parameters. +Future _handleGet(RequestContext context) async { + // Read dependencies provided by middleware + final modelName = context.read(); + final modelConfig = context.read>(); + final authenticatedUser = context.read(); + final requestId = context.read().id; - // Convert sortBy from camelCase to snake_case for the database query. - // This prevents errors where the client sends 'createdAt' and the database - // expects 'created_at'. - final sortBy = sortByParam != null ? _camelToSnake(sortByParam) : null; + // --- Parse Query Parameters --- + final params = context.request.uri.queryParameters; - SortOrder? sortOrder; - if (sortOrderRaw != null) { - if (sortOrderRaw == 'asc') { - sortOrder = SortOrder.asc; - } else if (sortOrderRaw == 'desc') { - sortOrder = SortOrder.desc; - } else { - throw const BadRequestException( - 'Invalid "sortOrder" parameter. Must be "asc" or "desc".', + // 1. Parse Filter (MongoDB-style) + Map? filter; + if (params.containsKey('filter')) { + try { + filter = jsonDecode(params['filter']!) as Map; + } on FormatException catch (e) { + throw BadRequestException( + 'Invalid "filter" parameter: Not valid JSON. $e', ); } } - final specificQueryForClient = {}; - final Set allowedKeys; - final receivedKeys = queryParams.keys - .where( - (k) => - k != 'model' && - k != 'startAfterId' && - k != 'limit' && - k != 'sortBy' && - k != 'sortOrder', - ) - .toSet(); - - switch (modelName) { - case 'headline': - allowedKeys = {'topics', 'sources', 'q'}; - final qValue = queryParams['q']; - if (qValue != null && qValue.isNotEmpty) { - specificQueryForClient['title_contains'] = qValue; - // specificQueryForClient['description_contains'] = qValue; // Removed - } else { - if (queryParams.containsKey('topics')) { - specificQueryForClient['topic.id_in'] = queryParams['topics']!; - } - if (queryParams.containsKey('sources')) { - specificQueryForClient['source.id_in'] = queryParams['sources']!; - } - } - case 'source': - allowedKeys = {'countries', 'sourceTypes', 'languages', 'q'}; - final qValue = queryParams['q']; - if (qValue != null && qValue.isNotEmpty) { - specificQueryForClient['name_contains'] = qValue; - // specificQueryForClient['description_contains'] = qValue; // Removed - } else { - if (queryParams.containsKey('countries')) { - specificQueryForClient['headquarters.iso_code_in'] = - queryParams['countries']!; - } - if (queryParams.containsKey('sourceTypes')) { - specificQueryForClient['source_type_in'] = - queryParams['sourceTypes']!; - } - if (queryParams.containsKey('languages')) { - specificQueryForClient['language_in'] = queryParams['languages']!; - } - } - case 'topic': - allowedKeys = {'q'}; - final qValue = queryParams['q']; - if (qValue != null && qValue.isNotEmpty) { - specificQueryForClient['name_contains'] = qValue; - // specificQueryForClient['description_contains'] = qValue; // Removed - } - case 'country': - allowedKeys = {'q'}; - final qValue = queryParams['q']; - if (qValue != null && qValue.isNotEmpty) { - specificQueryForClient['name_contains'] = qValue; - specificQueryForClient['iso_code_contains'] = qValue; // Added back - } - default: - // For other models, pass through all non-standard query params directly. - // No specific validation of allowed keys for these other models here. - // The client will attempt exact matches. - allowedKeys = receivedKeys; // Effectively allows all received keys - queryParams.forEach((key, value) { - if (key != 'model' && key != 'startAfterId' && key != 'limit') { - specificQueryForClient[key] = value; - } - }); + // 2. Parse Sort + List? sort; + if (params.containsKey('sort')) { + try { + sort = params['sort']! + .split(',') + .map((s) { + final parts = s.split(':'); + final field = parts[0]; + final order = (parts.length > 1 && parts[1] == 'desc') + ? SortOrder.desc + : SortOrder.asc; + return SortOption(field, order); + }) + .toList(); + } catch (e) { + throw BadRequestException( + 'Invalid "sort" parameter format. Use "field:order,field2:order".', + ); + } } - // Validate received keys against allowed keys for the specific models - if (modelName == 'headline' || - modelName == 'source' || - modelName == 'topic' || - modelName == 'country') { - for (final key in receivedKeys) { - if (!allowedKeys.contains(key)) { - throw BadRequestException( - 'Invalid query parameter "$key" for model "$modelName". ' - 'Allowed parameters are: ${allowedKeys.join(', ')}.', - ); - } - } + // 3. Parse Pagination + PaginationOptions? pagination; + if (params.containsKey('limit') || params.containsKey('cursor')) { + final limit = int.tryParse(params['limit'] ?? ''); + pagination = PaginationOptions(cursor: params['cursor'], limit: limit); } - PaginatedResponse paginatedResponse; - // ignore: prefer_final_locals - var userIdForRepoCall = modelConfig.getOwnerId != null - ? authenticatedUser.id - : null; + // --- Repository Call --- + final userIdForRepoCall = + (modelConfig.getOwnerId != null) ? authenticatedUser.id : null; + + dynamic responseData; - // Repository calls using specificQueryForClient + // The switch statement now only dispatches to the correct repository type. + // The query logic is handled by the repository/client. switch (modelName) { case 'headline': final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, + responseData = await repo.readAll( userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, + filter: filter, + sort: sort, + pagination: pagination, ); case 'topic': final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, + responseData = await repo.readAll( userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, + filter: filter, + sort: sort, + pagination: pagination, ); case 'source': final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, + responseData = await repo.readAll( userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, + filter: filter, + sort: sort, + pagination: pagination, ); case 'country': final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, + responseData = await repo.readAll( userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, + filter: filter, + sort: sort, + pagination: pagination, ); case 'user': final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, // Pass the potentially empty map - userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, - ); - case 'user_app_settings': - final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, + responseData = await repo.readAll( userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, - ); - case 'user_content_preferences': - final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, - userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, - ); - case 'remote_config': - final repo = context.read>(); - paginatedResponse = await repo.readAllByQuery( - specificQueryForClient, - userId: userIdForRepoCall, - startAfterId: startAfterId, - limit: limit, - sortBy: sortBy, - sortOrder: sortOrder, + filter: filter, + sort: sort, + pagination: pagination, ); default: throw OperationFailedException( - 'Unsupported model type "$modelName" reached data retrieval switch.', + 'Unsupported model type "$modelName" for GET all.', ); } - final finalFeedItems = paginatedResponse.items; + // --- Format and Return Response --- final metadata = ResponseMetadata( requestId: requestId, - timestamp: DateTime.now().toUtc(), // Use UTC for consistency + timestamp: DateTime.now().toUtc(), ); - // Wrap the PaginatedResponse in SuccessApiResponse with metadata - final successResponse = SuccessApiResponse>( - data: PaginatedResponse( - items: finalFeedItems, // Items are already dynamic - cursor: paginatedResponse.cursor, - hasMore: paginatedResponse.hasMore, - ), + final successResponse = SuccessApiResponse( + data: responseData, metadata: metadata, ); - // Need to provide the correct toJsonT for PaginatedResponse final responseJson = successResponse.toJson( - (paginated) => paginated.toJson( - (item) => (item as dynamic).toJson(), // Assuming all models have toJson + (paginated) => (paginated as PaginatedResponse).toJson( + (item) => (item as dynamic).toJson(), ), ); - // Return 200 OK with the wrapped and serialized response return Response.json(body: responseJson); } -// --- POST Handler --- -/// Handles POST requests: Creates a new item for the specified model. -/// Includes request metadata in response. -Future _handlePost( - RequestContext context, - String modelName, - ModelConfig modelConfig, - User authenticatedUser, - PermissionService permissionService, - String requestId, -) async { - // Authorization check is handled by authorizationMiddleware before this. +/// Handles POST requests: Creates a new item in a collection. +Future _handlePost(RequestContext context) async { + // Read dependencies from middleware + final modelName = context.read(); + final modelConfig = context.read>(); + final authenticatedUser = context.read(); + final requestId = context.read().id; + // --- Parse Body --- final requestBody = await context.request.json() as Map?; if (requestBody == null) { - // Throw BadRequestException to be caught by the errorHandler throw const BadRequestException('Missing or invalid request body.'); } - // Deserialize using ModelConfig's fromJson, catching TypeErrors - dynamic newItem; + dynamic itemToCreate; try { - newItem = modelConfig.fromJson(requestBody); + itemToCreate = modelConfig.fromJson(requestBody); } on TypeError catch (e) { - // Catch errors during deserialization (e.g., missing required fields) - // Include requestId in the server log - print('[ReqID: $requestId] Deserialization TypeError in POST /data: $e'); - // Throw BadRequestException to be caught by the errorHandler - throw const BadRequestException( - 'Invalid request body: Missing or invalid required field(s).', + throw BadRequestException( + 'Invalid request body: Missing or invalid required field(s). $e', ); } - // Determine userId for repository call based on ModelConfig (for data scoping/ownership enforcement) - String? userIdForRepoCall; - // If the model is user-owned, pass the authenticated user's ID to the repository - // for associating ownership during creation. Otherwise, pass null. - // We infer user-owned based on the presence of getOwnerId function. - if (modelConfig.getOwnerId != null) { - userIdForRepoCall = authenticatedUser.id; - } else { - userIdForRepoCall = null; - } + // --- Repository Call --- + final userIdForRepoCall = + (modelConfig.getOwnerId != null) ? authenticatedUser.id : null; - // Process based on model type dynamic createdItem; - - // Repository exceptions (like BadRequestException from create) will propagate - // up to the errorHandler. switch (modelName) { case 'headline': final repo = context.read>(); createdItem = await repo.create( - item: newItem as Headline, + item: itemToCreate as Headline, userId: userIdForRepoCall, ); case 'topic': final repo = context.read>(); createdItem = await repo.create( - item: newItem as Topic, + item: itemToCreate as Topic, userId: userIdForRepoCall, ); case 'source': final repo = context.read>(); createdItem = await repo.create( - item: newItem as Source, + item: itemToCreate as Source, userId: userIdForRepoCall, ); case 'country': final repo = context.read>(); createdItem = await repo.create( - item: newItem as Country, + item: itemToCreate as Country, userId: userIdForRepoCall, ); - case 'user': // Handle User model specifically if needed, or rely on generic - // User creation is typically handled by auth routes, not generic data POST. - // Throw Forbidden or BadRequest if attempted here. - throw const ForbiddenException( - 'User creation is not allowed via the generic data endpoint.', - ); - case 'user_app_settings': // New case for UserAppSettings - // Creation of UserAppSettings is handled by auth service, not generic data POST. - throw const ForbiddenException( - 'UserAppSettings creation is not allowed via the generic data endpoint.', - ); - case 'user_content_preferences': // New case for UserContentPreferences - // Creation of UserContentPreferences is handled by auth service, not generic data POST. - throw const ForbiddenException( - 'UserContentPreferences creation is not allowed via the generic data endpoint.', - ); - case 'remote_config': // New case for RemoteConfig (create by admin) + case 'remote_config': final repo = context.read>(); createdItem = await repo.create( - item: newItem as RemoteConfig, - userId: userIdForRepoCall, // userId should be null for AppConfig + item: itemToCreate as RemoteConfig, + userId: userIdForRepoCall, ); default: - // This case should ideally be caught by middleware, but added for safety - // Throw an exception to be caught by the errorHandler throw OperationFailedException( - 'Unsupported model type "$modelName" reached handler.', + 'Unsupported model type "$modelName" for POST.', ); } - // Create metadata including the request ID and current timestamp + // --- Format and Return Response --- final metadata = ResponseMetadata( requestId: requestId, - timestamp: DateTime.now().toUtc(), // Use UTC for consistency + timestamp: DateTime.now().toUtc(), ); - // Wrap the created item in SuccessApiResponse with metadata final successResponse = SuccessApiResponse( data: createdItem, metadata: metadata, ); - // Provide the correct toJsonT for the specific model type final responseJson = successResponse.toJson( - (item) => (item as dynamic).toJson(), // Assuming all models have toJson + (item) => (item as dynamic).toJson(), ); - // Return 201 Created with the wrapped and serialized response return Response.json(statusCode: HttpStatus.created, body: responseJson); -} - -/* -Simplified Strict Filtering Rules (ALL FILTERS ARE ANDed if present): - -1. Headlines (`model=headline`): - - Filterable by any combination (ANDed) of: - - `categories` (plural, comma-separated IDs, matching `headline.category.id`) - - `sources` (plural, comma-separated IDs, matching `headline.source.id`) - - `q` (free-text query, searching `headline.title` only) - - *No other filters (like `countries`) are allowed for headlines.* - -2. Sources (`model=source`): - - Filterable by any combination (ANDed) of: - - `countries` (plural, comma-separated ISO codes, matching `source.headquarters.iso_code`) - - `sourceTypes` (plural, comma-separated enum strings, matching `source.sourceType`) - - `languages` (plural, comma-separated language codes, matching `source.language`) - - `q` (free-text query, searching `source.name` only) - -3. Categories (`model=category`): - - Filterable __only__ by: - `q` (free-text query, searching `topic.name` - only) - -4. Countries (`model=country`): - - Filterable __only__ by: - `q` (free-text query, searching `country.name` - only) - ------- - -Explicitly Define Allowed Parameters per Model: When processing the request for a given `modelName`, the handler should have a predefined set of *allowed* query parameter keys for that specific model. - -- Example for `modelName == 'headline'`: - - Allowed keys: `categories`, `sources`, `q` (plus standard ones like `limit`, `startAfterId`). -- Example for `modelName == 'source'`: - - Allowed keys: `countries`, `sourceTypes`, `languages`, `q` (plus standard ones). -- And so on for `category` and `country`. - ------------------ TESTED FILTERS --------------- - -Model: `headline` - -1. Filter by single category: - - URL: `/api/v1/data?model=headline&topics=c1a2b3c4-d5e6-f789-0123-456789abcdef` - - Expected: Headlines with topic ID `c1a2b3c4-d5e6-f789-0123-456789abcdef`. - -2. Filter by multiple comma-separated categories (client-side `_in` implies OR for values): - - URL: `/api/v1/data?model=headline&topics=c1a2b3c4-d5e6-f789-0123-456789abcdef,c2b3c4d5-e6f7-a890-1234-567890abcdef` - - Expected: Headlines whose topic ID is *either* of the two provided. - -3. Filter by single source: - - URL: `/api/v1/data?model=headline&sources=s1a2b3c4-d5e6-f789-0123-456789abcdef` - - Expected: Headlines with source ID `s1a2b3c4-d5e6-f789-0123-456789abcdef`. - -4. Filter by multiple comma-separated sources (client-side `_in` implies OR for values): - - URL: `/api/v1/data?model=headline&sources=s1a2b3c4-d5e6-f789-0123-456789abcdef,s2b3c4d5-e6f7-a890-1234-567890abcdef` - - Expected: Headlines whose source ID is *either* of the two provided. - -5. Filter by a category AND a source: - - URL: `/api/v1/data?model=headline&topics=c1a2b3c4-d5e6-f789-0123-456789abcdef&sources=s1a2b3c4-d5e6-f789-0123-456789abcdef` - - Expected: Headlines matching *both* the topic ID AND the source ID. - -6. Filter by text query `q` (title only): - - URL: `/api/v1/data?model=headline&q=Dart` - - Expected: Headlines where "Dart" (case-insensitive) appears in the title. - -7. Filter by `q` AND `categories` (q should take precedence, categories ignored): - - URL: `/api/v1/data?model=headline&q=Flutter&topics=c1a2b3c4-d5e6-f789-0123-456789abcdef` - - Expected: Headlines matching `q=Flutter` (in title), ignoring the topic filter. - -8. Invalid parameter for headlines (e.g., `countries`): - - URL: `/api/v1/data?model=headline&countries=US` - - Expected: `400 Bad Request` with an error message about an invalid query parameter. - -Model: `source` - -9. Filter by single country (ISO code): - - URL: `/api/v1/data?model=source&countries=GB` - - Expected: Sources headquartered in 'GB'. - -10. Filter by multiple comma-separated countries (client-side `_in` implies OR for values): - - URL: `/api/v1/data?model=source&countries=US,GB` - - Expected: Sources headquartered in 'US' OR 'GB'. - -11. Filter by single `sourceType`: - - URL: `/api/v1/data?model=source&sourceTypes=blog` - - Expected: Sources of type 'blog'. - -12. Filter by multiple comma-separated `sourceTypes` (client-side `_in` implies OR for values): - - URL: `/api/v1/data?model=source&sourceTypes=blog,specializedPublisher` - - Expected: Sources of type 'blog' OR 'specializedPublisher'. - -13. Filter by single `language`: - - URL: `/api/v1/data?model=source&languages=en` - - Expected: Sources in 'en' language. - -14. Filter by combination (countries AND sourceTypes AND languages): - - URL: `/api/v1/data?model=source&countries=GB&sourceTypes=nationalNewsOutlet&languages=en` - - Expected: Sources matching all three criteria. - -15. Filter by text query `q` for sources (name only): - - URL: `/api/v1/data?model=source&q=Ventures` - - Expected: Sources where "Ventures" appears in the name. - -16. Filter by `q` AND `countries` for sources (`q` takes precedence): - - URL: `/api/v1/data?model=source&q=Official&countries=US` - - Expected: Sources matching `q=Official` (in name), ignoring the country filter. - -17. Invalid parameter for sources (e.g., `topics`): - - URL: `/api/v1/data?model=source&topics=topicId1` - - Expected: `400 Bad Request`. - -Model: `topic` - -18. Filter by text query `q` for categories (name only): - - URL: `/api/v1/data?model=topic&q=Mobile` - - Expected: Topics where "Mobile" appears in name. - -19. Invalid parameter for categories (e.g., `sources`): - - URL: `/api/v1/data?model=topic&sources=sourceId1` - - Expected: `400 Bad Request`. - -Model: `country` - -20. Filter by text query `q` for countries (name and iso_code): - - URL: `/api/v1/data?model=country&q=United` - - Expected: Countries where "United" appears in the name. - -21. Filter by text query `q` for countries (name and iso_code): - - URL: `/api/v1/data?model=country&q=US` - - Expected: Countries where "US" appears in the name OR the isoCode. - -22. Invalid parameter for countries (e.g., `categories`): - - URL: `/api/v1/data?model=country&topics=topicId1` - - Expected: `400 Bad Request`. -*/ +} \ No newline at end of file From 34a002951d2f67883ab331494878fb34e20ef375 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 12:16:39 +0100 Subject: [PATCH 04/23] refactor(api): adapt GET /data to use native document query - Rewrites the `_handleGet` method in the collection route handler to accept a single, JSON-encoded `filter` parameter. - Removes the complex logic for translating individual URL parameters into a query map. - Eliminates the `_camelToSnake` helper, as the API now passes native model field names to the data layer. - This change simplifies the API handler and aligns it with a document-native query model, making it more flexible and robust. --- routes/api/v1/data/index.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/routes/api/v1/data/index.dart b/routes/api/v1/data/index.dart index a4e4d73..01bfdd9 100644 --- a/routes/api/v1/data/index.dart +++ b/routes/api/v1/data/index.dart @@ -3,7 +3,6 @@ import 'dart:io'; import 'package:dart_frog/dart_frog.dart'; import 'package:ht_api/src/registry/model_registry.dart'; -import 'package:ht_api/src/services/user_preference_limit_service.dart'; import 'package:ht_data_repository/ht_data_repository.dart'; import 'package:ht_shared/ht_shared.dart'; From c065be483a2e07e9aba633229fbcc7182480d9df Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 12:32:35 +0100 Subject: [PATCH 05/23] feat(db): replace postgres seeding service with mongodb - Rewrites `DatabaseSeedingService` to work with a MongoDB `Db` instance. - Removes table creation logic, as MongoDB collections are schemaless. - Implements idempotent seeding using `bulkWrite` with `upsert` operations, preventing duplicate data on subsequent runs. - Correctly handles the conversion of string IDs from fixtures to MongoDB `ObjectId` for the `_id` field. - Ensures complex nested objects in fixtures are properly JSON-encoded before insertion. --- lib/src/config/database_connection.dart | 67 --- .../services/database_seeding_service.dart | 485 ++++-------------- 2 files changed, 114 insertions(+), 438 deletions(-) delete mode 100644 lib/src/config/database_connection.dart diff --git a/lib/src/config/database_connection.dart b/lib/src/config/database_connection.dart deleted file mode 100644 index 36e85c8..0000000 --- a/lib/src/config/database_connection.dart +++ /dev/null @@ -1,67 +0,0 @@ -import 'dart:async'; - -import 'package:ht_api/src/config/environment_config.dart'; -import 'package:logging/logging.dart'; -import 'package:postgres/postgres.dart'; - -/// A singleton class to manage a single, shared PostgreSQL database connection. -/// -/// This pattern ensures that the application establishes a connection to the -/// database only once and reuses it for all subsequent operations, which is -/// crucial for performance and resource management. -class DatabaseConnectionManager { - // Private constructor for the singleton pattern. - DatabaseConnectionManager._(); - - /// The single, global instance of the [DatabaseConnectionManager]. - static final instance = DatabaseConnectionManager._(); - - final _log = Logger('DatabaseConnectionManager'); - - // A completer to signal when the database connection is established. - final _completer = Completer(); - - /// Returns a future that completes with the established database connection. - /// - /// If the connection has not been initialized yet, it calls `init()` to - /// establish it. Subsequent calls will return the same connection future. - Future get connection => _completer.future; - - /// Initializes the database connection. - /// - /// This method is idempotent. It parses the database URL from the - /// environment, opens a connection to the PostgreSQL server, and completes - /// the `_completer` with the connection. It only performs the connection - /// logic on the very first call. - Future init() async { - if (_completer.isCompleted) { - _log.fine('Database connection already initializing/initialized.'); - return; - } - - _log.info('Initializing database connection...'); - final dbUri = Uri.parse(EnvironmentConfig.databaseUrl); - String? username; - String? password; - if (dbUri.userInfo.isNotEmpty) { - final parts = dbUri.userInfo.split(':'); - username = Uri.decodeComponent(parts.first); - if (parts.length > 1) { - password = Uri.decodeComponent(parts.last); - } - } - - final connection = await Connection.open( - Endpoint( - host: dbUri.host, - port: dbUri.port, - database: dbUri.path.substring(1), - username: username, - password: password, - ), - settings: const ConnectionSettings(sslMode: SslMode.require), - ); - _log.info('Database connection established successfully.'); - _completer.complete(connection); - } -} diff --git a/lib/src/services/database_seeding_service.dart b/lib/src/services/database_seeding_service.dart index cf85361..6b5dfcd 100644 --- a/lib/src/services/database_seeding_service.dart +++ b/lib/src/services/database_seeding_service.dart @@ -1,406 +1,149 @@ import 'dart:convert'; +import 'dart:io'; + import 'package:ht_shared/ht_shared.dart'; import 'package:logging/logging.dart'; -import 'package:postgres/postgres.dart'; +import 'package:mongo_dart/mongo_dart.dart'; /// {@template database_seeding_service} -/// A service responsible for initializing the database schema and seeding it -/// with initial data. +/// A service responsible for seeding the MongoDB database with initial data. /// -/// This service is intended to be run at application startup, particularly -/// in development environments or during the first run of a production instance -/// to set up the initial admin user and default configuration. +/// This service reads data from local JSON fixture files and uses `upsert` +/// operations to ensure that the seeding process is idempotent. It can be +/// run multiple times without creating duplicate documents. /// {@endtemplate} class DatabaseSeedingService { /// {@macro database_seeding_service} - const DatabaseSeedingService({ - required Connection connection, - required Logger log, - }) : _connection = connection, - _log = log; + const DatabaseSeedingService({required Db db, required Logger log}) + : _db = db, + _log = log; - final Connection _connection; + final Db _db; final Logger _log; - /// Creates all necessary tables in the database if they do not already exist. - /// - /// This method executes a series of `CREATE TABLE IF NOT EXISTS` statements - /// within a single transaction to ensure atomicity. - Future createTables() async { - _log.info('Starting database schema creation...'); - try { - // Manually handle the transaction with BEGIN/COMMIT/ROLLBACK. - await _connection.execute('BEGIN'); - - try { - _log.fine('Creating "users" table...'); - // All statements are executed on the main connection within the - // manual transaction. - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS users ( - id TEXT PRIMARY KEY, - email TEXT NOT NULL UNIQUE, - app_role TEXT NOT NULL, - dashboard_role TEXT NOT NULL, - feed_action_status JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ); - '''); - - _log.fine('Creating "remote_config" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS remote_config ( - id TEXT PRIMARY KEY, - user_preference_config JSONB NOT NULL, - ad_config JSONB NOT NULL, - account_action_config JSONB NOT NULL, - app_status JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - _log.fine('Creating "topics" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS topics ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - icon_url TEXT, - status TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - _log.fine('Creating "sources" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS sources ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - description TEXT, - url TEXT, - language TEXT, - status TEXT, - source_type TEXT, - headquarters_country_id TEXT REFERENCES countries(id), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - _log.fine('Creating "countries" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS countries ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - iso_code TEXT NOT NULL UNIQUE, - flag_url TEXT NOT NULL, - status TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - _log.fine('Creating "headlines" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS headlines ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - excerpt TEXT, - url TEXT, - image_url TEXT, - source_id TEXT REFERENCES sources(id), - topic_id TEXT REFERENCES topics(id), - event_country_id TEXT REFERENCES countries(id), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ, - status TEXT, - ); - '''); - - _log.fine('Creating "user_app_settings" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS user_app_settings ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, - display_settings JSONB NOT NULL, -- Nested object, stored as JSON - language TEXT NOT NULL, -- Simple string, stored as TEXT - feed_preferences JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - _log.fine('Creating "user_content_preferences" table...'); - await _connection.execute(''' - CREATE TABLE IF NOT EXISTS user_content_preferences ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, - followed_topics JSONB NOT NULL, - followed_sources JSONB NOT NULL, - followed_countries JSONB NOT NULL, - saved_headlines JSONB NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ - ); - '''); - - await _connection.execute('COMMIT'); - } catch (e) { - // If any query inside the transaction fails, roll back. - await _connection.execute('ROLLBACK'); - rethrow; // Re-throw the original error - } - _log.info('Database schema creation completed successfully.'); - } on Object catch (e, st) { - _log.severe('An error occurred during database schema creation.', e, st); - // Propagate as a standard exception for the server to handle. - throw OperationFailedException( - 'Failed to initialize database schema: $e', - ); - } + /// The main entry point for seeding all necessary data. + Future seedInitialData() async { + _log.info('Starting database seeding process...'); + await _seedCollection( + collectionName: 'countries', + fixturePath: 'lib/src/fixtures/countries.json', + ); + await _seedCollection( + collectionName: 'sources', + fixturePath: 'lib/src/fixtures/sources.json', + ); + await _seedCollection( + collectionName: 'topics', + fixturePath: 'lib/src/fixtures/topics.json', + ); + await _seedCollection( + collectionName: 'headlines', + fixturePath: 'lib/src/fixtures/headlines.json', + ); + await _seedInitialAdminAndConfig(); + _log.info('Database seeding process completed.'); } - /// Seeds the database with global fixture data (categories, sources, etc.). - /// - /// This method is idempotent, using `ON CONFLICT DO NOTHING` to prevent - /// errors if the data already exists. It runs within a single transaction. - Future seedGlobalFixtureData() async { - _log.info('Seeding global fixture data...'); + /// Seeds a specific collection from a given JSON fixture file. + Future _seedCollection({ + required String collectionName, + required String fixturePath, + }) async { + _log.info('Seeding collection: "$collectionName" from "$fixturePath"...'); try { - await _connection.execute('BEGIN'); - try { - // Seed Topics - _log.fine('Seeding topics...'); - for (final topic in topicsFixturesData) { - final params = topic.toJson() - ..putIfAbsent('description', () => null) - ..putIfAbsent('icon_url', () => null) - ..putIfAbsent('updated_at', () => null); + final collection = _db.collection(collectionName); + final file = File(fixturePath); + if (!await file.exists()) { + _log.warning('Fixture file not found: $fixturePath. Skipping.'); + return; + } - await _connection.execute( - Sql.named( - 'INSERT INTO topics (id, name, description, icon_url, ' - 'status, created_at, updated_at) VALUES (@id, @name, ' - '@description, @icon_url, @status, @created_at, @updated_at) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: params, - ); - } + final content = await file.readAsString(); + final documents = jsonDecode(content) as List; - // Seed Countries (must be done before sources and headlines) - _log.fine('Seeding countries...'); - for (final country in countriesFixturesData) { - final params = country.toJson() - ..putIfAbsent('updated_at', () => null); + if (documents.isEmpty) { + _log.info('No documents to seed for "$collectionName".'); + return; + } - await _connection.execute( - Sql.named( - 'INSERT INTO countries (id, name, iso_code, flag_url, ' - 'status, created_at, updated_at) VALUES (@id, @name, ' - '@iso_code, @flag_url, @status, @created_at, @updated_at) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: params, - ); - } + final bulk = collection.initializeUnorderedBulkOperation(); - // Seed Sources - _log.fine('Seeding sources...'); - for (final source in sourcesFixturesData) { - final params = source.toJson() - // The `headquarters` field in the model is a nested `Country` - // object. We extract its ID to store in the - // `headquarters_country_id` column and then remove the original - // nested object from the parameters to avoid a "superfluous - // variable" error. - ..['headquarters_country_id'] = source.headquarters.id - ..remove('headquarters') - ..putIfAbsent('description', () => null) - ..putIfAbsent('url', () => null) - ..putIfAbsent('language', () => null) - ..putIfAbsent('source_type', () => null) - ..putIfAbsent('updated_at', () => null); + for (final doc in documents) { + final docMap = doc as Map; + final id = docMap['id'] as String?; - await _connection.execute( - Sql.named( - 'INSERT INTO sources (id, name, description, url, language, ' - 'status, source_type, headquarters_country_id, ' - 'created_at, updated_at) VALUES (@id, @name, @description, @url, ' - '@language, @status, @source_type, ' - '@headquarters_country_id, @created_at, @updated_at) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: params, - ); + if (id == null || !ObjectId.isValidHexId(id)) { + _log.warning('Skipping document with invalid or missing ID: $doc'); + continue; } - // Seed Headlines - _log.fine('Seeding headlines...'); - for (final headline in headlinesFixturesData) { - final params = headline.toJson() - ..['source_id'] = headline.source.id - ..['topic_id'] = headline.topic.id - ..['event_country_id'] = headline.eventCountry.id - ..remove('source') - ..remove('topic') - ..remove('eventCountry') - ..putIfAbsent('excerpt', () => null) - ..putIfAbsent('updated_at', () => null) - ..putIfAbsent('image_url', () => null) - ..putIfAbsent('url', () => null); - - await _connection.execute( - Sql.named( - 'INSERT INTO headlines (id, title, excerpt, url, image_url, ' - 'source_id, topic_id, event_country_id, status, created_at, ' - 'updated_at) VALUES (@id, @title, @excerpt, @url, @image_url, ' - '@source_id, @topic_id, @event_country_id, @status, @created_at, @updated_at) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: params, - ); - } + final objectId = ObjectId.fromHexString(id); + // Remove the string 'id' field and use '_id' with ObjectId + docMap.remove('id'); - await _connection.execute('COMMIT'); - _log.info('Global fixture data seeding completed successfully.'); - } catch (e) { - await _connection.execute('ROLLBACK'); - rethrow; + bulk.find({'_id': objectId}).upsert().replaceOne(docMap); } - } on Object catch (e, st) { + + final result = await bulk.execute(); + _log.info( + 'Seeding for "$collectionName" complete. ' + 'Upserted: ${result.nUpserted}, Modified: ${result.nModified}.', + ); + } on Exception catch (e, s) { _log.severe( - 'An error occurred during global fixture data seeding.', + 'Failed to seed collection "$collectionName" from "$fixturePath".', e, - st, + s, ); - throw OperationFailedException('Failed to seed global fixture data: $e'); + // Re-throwing to halt the startup process if seeding fails. + rethrow; } } - /// Seeds the database with the initial RemoteConfig and the default admin user. - /// - /// This method is idempotent, using `ON CONFLICT DO NOTHING` to prevent - /// errors if the data already exists. It runs within a single transaction. - Future seedInitialAdminAndConfig() async { - _log.info('Seeding initial RemoteConfig and admin user...'); + /// Seeds the initial admin user and remote config document. + Future _seedInitialAdminAndConfig() async { + _log.info('Seeding initial admin user and remote config...'); try { - await _connection.execute('BEGIN'); - try { - // Seed RemoteConfig - _log.fine('Seeding RemoteConfig...'); - const remoteConfig = remoteConfigFixtureData; - // The `remote_config` table has multiple JSONB columns. We must - // provide an explicit map with JSON-encoded values to avoid a - // "superfluous variables" error from the postgres driver. - await _connection.execute( - Sql.named( - 'INSERT INTO remote_config (id, user_preference_config, ad_config, ' - 'account_action_config, app_status) VALUES (@id, ' - '@user_preference_config, @ad_config, @account_action_config, ' - '@app_status) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: { - 'id': remoteConfig.id, - 'user_preference_config': jsonEncode( - remoteConfig.userPreferenceConfig.toJson(), - ), - 'ad_config': jsonEncode(remoteConfig.adConfig.toJson()), - 'account_action_config': jsonEncode( - remoteConfig.accountActionConfig.toJson(), - ), - 'app_status': jsonEncode(remoteConfig.appStatus.toJson()), - }, - ); - - // Seed Admin User - _log.fine('Seeding admin user...'); - // Find the admin user in the fixture data. - final adminUser = usersFixturesData.firstWhere( - (user) => user.dashboardRole == DashboardUserRole.admin, - orElse: () => throw StateError('Admin user not found in fixtures.'), - ); - - // The `users` table has specific columns for roles and status. - await _connection.execute( - Sql.named( - 'INSERT INTO users (id, email, app_role, dashboard_role, ' - 'feed_action_status) VALUES (@id, @email, @app_role, ' - '@dashboard_role, @feed_action_status) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: () { - final params = adminUser.toJson(); - params['feed_action_status'] = jsonEncode( - params['feed_action_status'], - ); - return params; - }(), - ); - - // Seed default settings and preferences for the admin user. - final adminSettings = userAppSettingsFixturesData.firstWhere( - (settings) => settings.id == adminUser.id, - ); - final adminPreferences = userContentPreferencesFixturesData.firstWhere( - (prefs) => prefs.id == adminUser.id, - ); - - await _connection.execute( - Sql.named( - 'INSERT INTO user_app_settings (id, user_id, display_settings, ' - 'language, feed_preferences) VALUES (@id, @user_id, ' - '@display_settings, @language, @feed_preferences) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: () { - final params = adminSettings.toJson(); - params['user_id'] = adminUser.id; - params['display_settings'] = jsonEncode(params['display_settings']); - params['feed_preferences'] = jsonEncode(params['feed_preferences']); - return params; - }(), - ); - - await _connection.execute( - Sql.named( - 'INSERT INTO user_content_preferences (id, user_id, ' - 'followed_topics, followed_sources, followed_countries, ' - 'saved_headlines) VALUES (@id, @user_id, @followed_topics, ' - '@followed_sources, @followed_countries, @saved_headlines) ' - 'ON CONFLICT (id) DO NOTHING', - ), - parameters: () { - final params = adminPreferences.toJson(); - params['user_id'] = adminUser.id; - params['followed_topics'] = jsonEncode(params['followed_topics']); - params['followed_sources'] = jsonEncode(params['followed_sources']); - params['followed_countries'] = jsonEncode( - params['followed_countries'], - ); - params['saved_headlines'] = jsonEncode(params['saved_headlines']); - return params; - }(), - ); - - await _connection.execute('COMMIT'); - _log.info('Initial RemoteConfig and admin user seeding completed.'); - } catch (e) { - await _connection.execute('ROLLBACK'); - rethrow; - } - } on Object catch (e, st) { - _log.severe( - 'An error occurred during initial admin/config seeding.', - e, - st, + // --- Seed Admin User --- + final usersCollection = _db.collection('users'); + final adminUser = User.fromJson(adminUserFixture); + final adminDoc = adminUser.toJson() + ..['app_role'] = adminUser.appRole.name + ..['dashboard_role'] = adminUser.dashboardRole.name + ..['feed_action_status'] = jsonEncode(adminUser.feedActionStatus) + ..remove('id'); + + await usersCollection.updateOne( + where.id(ObjectId.fromHexString(adminUser.id)), + modify.set( + 'email', + adminDoc['email'], + ).setAll(adminDoc), // Use setAll to add/update all fields + upsert: true, ); - throw OperationFailedException( - 'Failed to seed initial admin/config data: $e', + _log.info('Admin user seeded successfully.'); + + // --- Seed Remote Config --- + final remoteConfigCollection = _db.collection('remote_config'); + final remoteConfig = RemoteConfig.fromJson(remoteConfigFixture); + final remoteConfigDoc = remoteConfig.toJson() + ..['user_preference_limits'] = + jsonEncode(remoteConfig.userPreferenceConfig.toJson()) + ..['ad_config'] = jsonEncode(remoteConfig.adConfig.toJson()) + ..['account_action_config'] = + jsonEncode(remoteConfig.accountActionConfig.toJson()) + ..['app_status'] = jsonEncode(remoteConfig.appStatus.toJson()) + ..remove('id'); + + await remoteConfigCollection.updateOne( + where.id(ObjectId.fromHexString(remoteConfig.id)), + modify.setAll(remoteConfigDoc), + upsert: true, ); + _log.info('Remote config seeded successfully.'); + } on Exception catch (e, s) { + _log.severe('Failed to seed admin user or remote config.', e, s); + rethrow; } } -} +} \ No newline at end of file From fe51a1e9bdee8048cf024ab9d3120de9b2d690c3 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 13:33:20 +0100 Subject: [PATCH 06/23] feat(db): replace postgres seeding service with mongodb - Rewrites `DatabaseSeedingService` to work with a MongoDB `Db` instance. - Removes table creation logic, as MongoDB collections are schemaless. - Implements idempotent seeding using `bulkWrite` with `upsert` operations, preventing duplicate data on subsequent runs. - Correctly handles the conversion of string IDs from fixtures to MongoDB `ObjectId` for the `_id` field. - Ensures complex nested objects in fixtures are properly JSON-encoded before insertion. --- .../services/database_seeding_service.dart | 160 +++++++----------- 1 file changed, 64 insertions(+), 96 deletions(-) diff --git a/lib/src/services/database_seeding_service.dart b/lib/src/services/database_seeding_service.dart index 6b5dfcd..fbd0103 100644 --- a/lib/src/services/database_seeding_service.dart +++ b/lib/src/services/database_seeding_service.dart @@ -1,6 +1,3 @@ -import 'dart:convert'; -import 'dart:io'; - import 'package:ht_shared/ht_shared.dart'; import 'package:logging/logging.dart'; import 'package:mongo_dart/mongo_dart.dart'; @@ -8,15 +5,15 @@ import 'package:mongo_dart/mongo_dart.dart'; /// {@template database_seeding_service} /// A service responsible for seeding the MongoDB database with initial data. /// -/// This service reads data from local JSON fixture files and uses `upsert` -/// operations to ensure that the seeding process is idempotent. It can be -/// run multiple times without creating duplicate documents. +/// This service reads data from predefined fixture lists in `ht_shared` and +/// uses `upsert` operations to ensure that the seeding process is idempotent. +/// It can be run multiple times without creating duplicate documents. /// {@endtemplate} class DatabaseSeedingService { /// {@macro database_seeding_service} const DatabaseSeedingService({required Db db, required Logger log}) - : _db = db, - _log = log; + : _db = db, + _log = log; final Db _db; final Logger _log; @@ -24,126 +21,97 @@ class DatabaseSeedingService { /// The main entry point for seeding all necessary data. Future seedInitialData() async { _log.info('Starting database seeding process...'); - await _seedCollection( + + await _seedCollection( collectionName: 'countries', - fixturePath: 'lib/src/fixtures/countries.json', + fixtureData: countriesFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), ); - await _seedCollection( + await _seedCollection( collectionName: 'sources', - fixturePath: 'lib/src/fixtures/sources.json', + fixtureData: sourcesFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), ); - await _seedCollection( + await _seedCollection( collectionName: 'topics', - fixturePath: 'lib/src/fixtures/topics.json', + fixtureData: topicsFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), ); - await _seedCollection( + await _seedCollection( collectionName: 'headlines', - fixturePath: 'lib/src/fixtures/headlines.json', + fixtureData: headlinesFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), + ); + await _seedCollection( + collectionName: 'users', + fixtureData: usersFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), + ); + await _seedCollection( + collectionName: 'remote_configs', + fixtureData: remoteConfigsFixturesData, + getId: (item) => item.id, + toJson: (item) => item.toJson(), ); - await _seedInitialAdminAndConfig(); + _log.info('Database seeding process completed.'); } - /// Seeds a specific collection from a given JSON fixture file. - Future _seedCollection({ + /// Seeds a specific collection from a given list of fixture data. + Future _seedCollection({ required String collectionName, - required String fixturePath, + required List fixtureData, + required String Function(T) getId, + required Map Function(T) toJson, }) async { - _log.info('Seeding collection: "$collectionName" from "$fixturePath"...'); + _log.info('Seeding collection: "$collectionName"...'); try { - final collection = _db.collection(collectionName); - final file = File(fixturePath); - if (!await file.exists()) { - _log.warning('Fixture file not found: $fixturePath. Skipping.'); - return; - } - - final content = await file.readAsString(); - final documents = jsonDecode(content) as List; - - if (documents.isEmpty) { + if (fixtureData.isEmpty) { _log.info('No documents to seed for "$collectionName".'); return; } - final bulk = collection.initializeUnorderedBulkOperation(); + final collection = _db.collection(collectionName); + final operations = >[]; - for (final doc in documents) { - final docMap = doc as Map; - final id = docMap['id'] as String?; + for (final item in fixtureData) { + final id = getId(item); - if (id == null || !ObjectId.isValidHexId(id)) { - _log.warning('Skipping document with invalid or missing ID: $doc'); + if (!ObjectId.isValidHexId(id)) { + _log.warning('Skipping document with invalid ID format: $id'); continue; } final objectId = ObjectId.fromHexString(id); - // Remove the string 'id' field and use '_id' with ObjectId - docMap.remove('id'); + final document = toJson(item)..remove('id'); + + operations.add({ + 'replaceOne': { + 'filter': {'_id': objectId}, + 'replacement': document, + 'upsert': true, + }, + }); + } - bulk.find({'_id': objectId}).upsert().replaceOne(docMap); + if (operations.isEmpty) { + _log.info('No valid documents to write for "$collectionName".'); + return; } - final result = await bulk.execute(); + final result = await collection.bulkWrite(operations); _log.info( 'Seeding for "$collectionName" complete. ' 'Upserted: ${result.nUpserted}, Modified: ${result.nModified}.', ); } on Exception catch (e, s) { - _log.severe( - 'Failed to seed collection "$collectionName" from "$fixturePath".', - e, - s, - ); - // Re-throwing to halt the startup process if seeding fails. - rethrow; - } - } - - /// Seeds the initial admin user and remote config document. - Future _seedInitialAdminAndConfig() async { - _log.info('Seeding initial admin user and remote config...'); - try { - // --- Seed Admin User --- - final usersCollection = _db.collection('users'); - final adminUser = User.fromJson(adminUserFixture); - final adminDoc = adminUser.toJson() - ..['app_role'] = adminUser.appRole.name - ..['dashboard_role'] = adminUser.dashboardRole.name - ..['feed_action_status'] = jsonEncode(adminUser.feedActionStatus) - ..remove('id'); - - await usersCollection.updateOne( - where.id(ObjectId.fromHexString(adminUser.id)), - modify.set( - 'email', - adminDoc['email'], - ).setAll(adminDoc), // Use setAll to add/update all fields - upsert: true, - ); - _log.info('Admin user seeded successfully.'); - - // --- Seed Remote Config --- - final remoteConfigCollection = _db.collection('remote_config'); - final remoteConfig = RemoteConfig.fromJson(remoteConfigFixture); - final remoteConfigDoc = remoteConfig.toJson() - ..['user_preference_limits'] = - jsonEncode(remoteConfig.userPreferenceConfig.toJson()) - ..['ad_config'] = jsonEncode(remoteConfig.adConfig.toJson()) - ..['account_action_config'] = - jsonEncode(remoteConfig.accountActionConfig.toJson()) - ..['app_status'] = jsonEncode(remoteConfig.appStatus.toJson()) - ..remove('id'); - - await remoteConfigCollection.updateOne( - where.id(ObjectId.fromHexString(remoteConfig.id)), - modify.setAll(remoteConfigDoc), - upsert: true, - ); - _log.info('Remote config seeded successfully.'); - } on Exception catch (e, s) { - _log.severe('Failed to seed admin user or remote config.', e, s); + _log.severe('Failed to seed collection "$collectionName".', e, s); rethrow; } } -} \ No newline at end of file +} From 1bca9ab00a7331d43a36bc4a295756ac51d6f7d9 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 13:34:35 +0100 Subject: [PATCH 07/23] refactor(deps): remove postgres dependencies - Removes `ht_data_postgres` and `postgres` from the `pubspec.yaml`. - This is a key step in migrating the data layer from PostgreSQL to MongoDB, severing the project's dependency on the old database implementation. --- pubspec.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pubspec.yaml b/pubspec.yaml index 2339931..fb6e8d2 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -16,9 +16,6 @@ dependencies: ht_data_mongodb: git: url: https://github.com/headlines-toolkit/ht-data-mongodb.git - ht_data_postgres: - git: - url: https://github.com/headlines-toolkit/ht-data-postgres.git ht_data_repository: git: url: https://github.com/headlines-toolkit/ht-data-repository.git @@ -41,7 +38,6 @@ dependencies: logging: ^1.3.0 meta: ^1.16.0 mongo_dart: ^0.10.5 - postgres: ^3.5.6 shelf_cors_headers: ^0.1.5 uuid: ^4.5.1 From bfb3e5f953bf495d36a4cab78cf58ddbd7cb27ae Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 13:42:11 +0100 Subject: [PATCH 08/23] feat(db): migrate dependency injection to mongodb - Rewrites `AppDependencies` to use `MongoDbConnectionManager` instead of the PostgreSQL equivalent. - Initializes the MongoDB connection and runs the new `DatabaseSeedingService`. - Instantiates `HtDataMongodb` clients for all data models. - Wires up all `HtDataRepository` instances and application services to use the new MongoDB-backed data layer. - This completes the core dependency injection part of the migration. --- lib/src/config/app_dependencies.dart | 319 ++++++++++----------------- 1 file changed, 116 insertions(+), 203 deletions(-) diff --git a/lib/src/config/app_dependencies.dart b/lib/src/config/app_dependencies.dart index 7054ff0..0342396 100644 --- a/lib/src/config/app_dependencies.dart +++ b/lib/src/config/app_dependencies.dart @@ -1,7 +1,4 @@ -import 'dart:async'; -import 'dart:convert'; - -import 'package:ht_api/src/config/database_connection.dart'; +import 'package:ht_api/src/config/environment_config.dart'; import 'package:ht_api/src/rbac/permission_service.dart'; import 'package:ht_api/src/services/auth_service.dart'; import 'package:ht_api/src/services/auth_token_service.dart'; @@ -12,225 +9,162 @@ import 'package:ht_api/src/services/jwt_auth_token_service.dart'; import 'package:ht_api/src/services/token_blacklist_service.dart'; import 'package:ht_api/src/services/user_preference_limit_service.dart'; import 'package:ht_api/src/services/verification_code_storage_service.dart'; -import 'package:ht_data_client/ht_data_client.dart'; -import 'package:ht_data_postgres/ht_data_postgres.dart'; +import 'package:ht_data_mongodb/ht_data_mongodb.dart'; import 'package:ht_data_repository/ht_data_repository.dart'; import 'package:ht_email_inmemory/ht_email_inmemory.dart'; import 'package:ht_email_repository/ht_email_repository.dart'; import 'package:ht_shared/ht_shared.dart'; import 'package:logging/logging.dart'; -import 'package:postgres/postgres.dart'; import 'package:uuid/uuid.dart'; -/// A singleton class to manage all application dependencies. -/// -/// This class follows a lazy initialization pattern. Dependencies are created -/// only when the `init()` method is first called, typically triggered by the -/// first incoming request. A `Completer` ensures that subsequent requests -/// await the completion of the initial setup. +/// {@template app_dependencies} +/// A singleton class responsible for initializing and providing all application +/// dependencies, such as database connections, repositories, and services. +/// {@endtemplate} class AppDependencies { + /// Private constructor for the singleton pattern. AppDependencies._(); - /// The single, global instance of the [AppDependencies]. - static final instance = AppDependencies._(); + /// The single, static instance of this class. + static final AppDependencies _instance = AppDependencies._(); + + /// Provides access to the singleton instance. + static AppDependencies get instance => _instance; + bool _isInitialized = false; final _log = Logger('AppDependencies'); - final _completer = Completer(); - // --- Repositories --- - /// A repository for managing [Headline] data. - late final HtDataRepository headlineRepository; + // --- Late-initialized fields for all dependencies --- - /// A repository for managing [Topic] data. - late final HtDataRepository topicRepository; + // Database + late final MongoDbConnectionManager _mongoDbConnectionManager; - /// A repository for managing [Source] data. + // Repositories + late final HtDataRepository headlineRepository; + late final HtDataRepository topicRepository; late final HtDataRepository sourceRepository; - - /// A repository for managing [Country] data. late final HtDataRepository countryRepository; - - /// A repository for managing [User] data. late final HtDataRepository userRepository; - - /// A repository for managing [UserAppSettings] data. late final HtDataRepository userAppSettingsRepository; - - /// A repository for managing [UserContentPreferences] data. late final HtDataRepository - userContentPreferencesRepository; - - /// A repository for managing the global [RemoteConfig] data. + userContentPreferencesRepository; late final HtDataRepository remoteConfigRepository; - - // --- Services --- - /// A service for sending emails. late final HtEmailRepository emailRepository; - /// A service for managing a blacklist of invalidated authentication tokens. + // Services late final TokenBlacklistService tokenBlacklistService; - - /// A service for generating and validating authentication tokens. late final AuthTokenService authTokenService; - - /// A service for storing and validating one-time verification codes. late final VerificationCodeStorageService verificationCodeStorageService; - - /// A service that orchestrates authentication logic. late final AuthService authService; - - /// A service for calculating and providing a summary for the dashboard. late final DashboardSummaryService dashboardSummaryService; - - /// A service for checking user permissions. late final PermissionService permissionService; - - /// A service for enforcing limits on user content preferences. late final UserPreferenceLimitService userPreferenceLimitService; /// Initializes all application dependencies. /// - /// This method is idempotent. It performs the full initialization only on - /// the first call. Subsequent calls will await the result of the first one. - Future init() { - if (_completer.isCompleted) { - _log.fine('Dependencies already initializing/initialized.'); - return _completer.future; - } + /// This method is idempotent; it will only run the initialization logic once. + Future init() async { + if (_isInitialized) return; _log.info('Initializing application dependencies...'); - _init() - .then((_) { - _log.info('Application dependencies initialized successfully.'); - _completer.complete(); - }) - .catchError((Object e, StackTrace s) { - _log.severe('Failed to initialize application dependencies.', e, s); - _completer.completeError(e, s); - }); - - return _completer.future; - } - Future _init() async { - // 1. Establish Database Connection. - await DatabaseConnectionManager.instance.init(); - final connection = await DatabaseConnectionManager.instance.connection; + // 1. Initialize Database Connection + _mongoDbConnectionManager = MongoDbConnectionManager(); + await _mongoDbConnectionManager.init(EnvironmentConfig.databaseUrl); + _log.info('MongoDB connection established.'); - // 2. Run Database Seeding. + // 2. Seed Database final seedingService = DatabaseSeedingService( - connection: connection, - log: _log, + db: _mongoDbConnectionManager.db, + log: Logger('DatabaseSeedingService'), ); - await seedingService.createTables(); - await seedingService.seedGlobalFixtureData(); - await seedingService.seedInitialAdminAndConfig(); - - // 3. Initialize Repositories. - headlineRepository = _createRepository( - connection, - 'headlines', - // The HtDataPostgresClient returns DateTime objects from TIMESTAMPTZ - // columns. The Headline.fromJson factory expects ISO 8601 strings. - // This handler converts them before deserialization. - (json) => Headline.fromJson(_convertTimestampsToString(json)), - (headline) => headline.toJson() - ..['source_id'] = headline.source.id - ..['topic_id'] = headline.topic.id - ..['event_country_id'] = headline.eventCountry.id - ..remove('source') - ..remove('topic') - ..remove('eventCountry'), + await seedingService.seedInitialData(); + _log.info('Database seeding complete.'); + + // 3. Initialize Data Clients (MongoDB implementation) + final headlineClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'headlines', + fromJson: Headline.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final topicClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'topics', + fromJson: Topic.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - topicRepository = _createRepository( - connection, - 'topics', - (json) => Topic.fromJson(_convertTimestampsToString(json)), - (topic) => topic.toJson(), + final sourceClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'sources', + fromJson: Source.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - sourceRepository = _createRepository( - connection, - 'sources', - (json) => Source.fromJson(_convertTimestampsToString(json)), - (source) => source.toJson() - ..['headquarters_country_id'] = source.headquarters.id - ..remove('headquarters'), + final countryClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'countries', + fromJson: Country.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - countryRepository = _createRepository( - connection, - 'countries', - (json) => Country.fromJson(_convertTimestampsToString(json)), - (country) => country.toJson(), + final userClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'users', + fromJson: User.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - userRepository = _createRepository( - connection, - 'users', - (json) => User.fromJson(_convertTimestampsToString(json)), - (user) { - final json = user.toJson(); - // Convert enums to their string names for the database. - json['app_role'] = user.appRole.name; - json['dashboard_role'] = user.dashboardRole.name; - // The `feed_action_status` map must be JSON encoded for the JSONB column. - json['feed_action_status'] = jsonEncode(json['feed_action_status']); - return json; - }, + final userAppSettingsClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'user_app_settings', + fromJson: UserAppSettings.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - userAppSettingsRepository = _createRepository( - connection, - 'user_app_settings', - UserAppSettings.fromJson, - (settings) { - final json = settings.toJson(); - // These fields are complex objects and must be JSON encoded for the DB. - json['display_settings'] = jsonEncode(json['display_settings']); - json['feed_preferences'] = jsonEncode(json['feed_preferences']); - return json; - }, + final userContentPreferencesClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'user_content_preferences', + fromJson: UserContentPreferences.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - userContentPreferencesRepository = _createRepository( - connection, - 'user_content_preferences', - UserContentPreferences.fromJson, - (preferences) { - final json = preferences.toJson(); - // These fields are lists of complex objects and must be JSON encoded. - json['followed_topics'] = jsonEncode(json['followed_topics']); - json['followed_sources'] = jsonEncode(json['followed_sources']); - json['followed_countries'] = jsonEncode(json['followed_countries']); - json['saved_headlines'] = jsonEncode(json['saved_headlines']); - return json; - }, + final remoteConfigClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'remote_configs', + fromJson: RemoteConfig.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), ); - remoteConfigRepository = _createRepository( - connection, - 'remote_config', - (json) => RemoteConfig.fromJson(_convertTimestampsToString(json)), - (config) { - final json = config.toJson(); - // All nested config objects must be JSON encoded for JSONB columns. - json['user_preference_limits'] = jsonEncode( - json['user_preference_limits'], - ); - json['ad_config'] = jsonEncode(json['ad_config']); - json['account_action_config'] = jsonEncode( - json['account_action_config'], - ); - json['app_status'] = jsonEncode(json['app_status']); - return json; - }, + + // 4. Initialize Repositories + headlineRepository = HtDataRepository(dataClient: headlineClient); + topicRepository = HtDataRepository(dataClient: topicClient); + sourceRepository = HtDataRepository(dataClient: sourceClient); + countryRepository = HtDataRepository(dataClient: countryClient); + userRepository = HtDataRepository(dataClient: userClient); + userAppSettingsRepository = + HtDataRepository(dataClient: userAppSettingsClient); + userContentPreferencesRepository = + HtDataRepository(dataClient: userContentPreferencesClient); + remoteConfigRepository = HtDataRepository(dataClient: remoteConfigClient); + + final emailClient = HtEmailInMemoryClient( + logger: Logger('HtEmailInMemoryClient'), ); + emailRepository = HtEmailRepository(emailClient: emailClient); - // 4. Initialize Services. - emailRepository = const HtEmailRepository( - emailClient: HtEmailInMemoryClient(), + // 5. Initialize Services + tokenBlacklistService = InMemoryTokenBlacklistService( + log: Logger('InMemoryTokenBlacklistService'), ); - tokenBlacklistService = InMemoryTokenBlacklistService(log: _log); authTokenService = JwtAuthTokenService( userRepository: userRepository, blacklistService: tokenBlacklistService, uuidGenerator: const Uuid(), - log: _log, + log: Logger('JwtAuthTokenService'), ); verificationCodeStorageService = InMemoryVerificationCodeStorageService(); authService = AuthService( @@ -241,7 +175,7 @@ class AppDependencies { userAppSettingsRepository: userAppSettingsRepository, userContentPreferencesRepository: userContentPreferencesRepository, uuidGenerator: const Uuid(), - log: _log, + log: Logger('AuthService'), ); dashboardSummaryService = DashboardSummaryService( headlineRepository: headlineRepository, @@ -251,40 +185,19 @@ class AppDependencies { permissionService = const PermissionService(); userPreferenceLimitService = DefaultUserPreferenceLimitService( remoteConfigRepository: remoteConfigRepository, - log: _log, + log: Logger('DefaultUserPreferenceLimitService'), ); - } - HtDataRepository _createRepository( - Connection connection, - String tableName, - FromJson fromJson, - ToJson toJson, - ) { - return HtDataRepository( - dataClient: HtDataPostgresClient( - connection: connection, - tableName: tableName, - fromJson: fromJson, - toJson: toJson, - log: _log, - ), - ); + _isInitialized = true; + _log.info('Application dependencies initialized successfully.'); } - /// Converts DateTime values in a JSON map to ISO 8601 strings. - /// - /// The postgres driver returns DateTime objects for TIMESTAMPTZ columns, - /// but our models' `fromJson` factories expect ISO 8601 strings. This - /// utility function performs the conversion for known timestamp fields. - Map _convertTimestampsToString(Map json) { - const timestampKeys = {'created_at', 'updated_at'}; - final newJson = Map.from(json); - for (final key in timestampKeys) { - if (newJson[key] is DateTime) { - newJson[key] = (newJson[key] as DateTime).toIso8601String(); - } - } - return newJson; + /// Disposes of resources, such as closing the database connection. + Future dispose() async { + if (!_isInitialized) return; + await _mongoDbConnectionManager.close(); + tokenBlacklistService.dispose(); + _isInitialized = false; + _log.info('Application dependencies disposed.'); } -} +} \ No newline at end of file From bf6f59d891e9cf0bb7aac9689702e8e85198c334 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 13:43:06 +0100 Subject: [PATCH 09/23] docs: update readme for mongodb migration - Updates the prerequisites in the README to specify MongoDB instead of PostgreSQL. - Changes the example `DATABASE_URL` to a MongoDB connection string. - Rewrites the description of the server startup process to reflect idempotent seeding of MongoDB collections. - Removes a duplicate bullet point for clarity. --- README.md | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 36d50e4..93e9aae 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,6 @@ management dashboard](https://github.com/headlines-toolkit/ht-dashboard). behavior—including ad frequency, feature flags, and maintenance status—without requiring a client-side update. -* 💾 **Robust Data Management:** Securely manage core news data (headlines, - topics, sources) through a well-structured API that supports flexible - querying and sorting for dynamic content presentation. - * 📊 **Dynamic Dashboard Summary:** Access real-time, aggregated metrics on key data points like total headlines, topics, and sources, providing an at-a-glance overview for administrative dashboards. @@ -75,7 +71,7 @@ for more details. 1. **Prerequisites:** * Dart SDK (`>=3.0.0`) - * PostgreSQL (`>=14.0` recommended) + * MongoDB (`>=5.0` recommended) * Dart Frog CLI (`dart pub global activate dart_frog_cli`) 2. **Configuration:** @@ -85,7 +81,7 @@ for more details. Create a `.env` file in the root of the project or export the variable in your shell: ``` - DATABASE_URL="postgres://user:password@localhost:5432/ht_api_db" + DATABASE_URL="mongodb://user:password@localhost:27017/ht_api_db" ``` 3. **Clone the repository:** @@ -101,11 +97,10 @@ for more details. ```bash dart_frog dev ``` - The API will typically be available at `http://localhost:8080`. On the - first startup, the server will connect to your PostgreSQL database, create the - necessary tables, and seed them with initial fixture data. This process is - non-destructive; it uses `CREATE TABLE IF NOT EXISTS` and `INSERT ... ON - CONFLICT DO NOTHING` to avoid overwriting existing tables or data. + The API will typically be available at `http://localhost:8080`. On startup, + the server will connect to your MongoDB database and seed it with initial + fixture data. This seeding process is idempotent (using `upsert` + operations), so it can be run multiple times without creating duplicates. **Note on Web Client Integration (CORS):** To allow web applications (like From c6877bc6cffc41e16b66d4435039c02d1f22aaaf Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 13:51:32 +0100 Subject: [PATCH 10/23] refactor(auth): use readAll with filter in AuthService - Replaces the obsolete `readAllByQuery` method with the new `readAll` method in `AuthService`. - Passes the query conditions as a `filter` map, aligning the service with the new document-based data access pattern. - This change completes the service-level refactoring for the MongoDB migration. --- lib/src/services/auth_service.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/src/services/auth_service.dart b/lib/src/services/auth_service.dart index 05f53bd..b015d61 100644 --- a/lib/src/services/auth_service.dart +++ b/lib/src/services/auth_service.dart @@ -411,8 +411,9 @@ class AuthService { try { // 1. Check if emailToLink is already used by another permanent user. - final query = {'email': emailToLink}; - final existingUsersResponse = await _userRepository.readAllByQuery(query); + final existingUsersResponse = await _userRepository.readAll( + filter: {'email': emailToLink}, + ); // Filter for permanent users (not guests) that are not the current user. final conflictingPermanentUsers = existingUsersResponse.items.where( @@ -612,8 +613,9 @@ class AuthService { /// Re-throws any [HtHttpException] from the repository. Future _findUserByEmail(String email) async { try { - final query = {'email': email}; - final response = await _userRepository.readAllByQuery(query); + final response = await _userRepository.readAll( + filter: {'email': email}, + ); if (response.items.isNotEmpty) { return response.items.first; } From 38853eaa3591b72bd675a0434deda34c4c783561 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 14:12:05 +0100 Subject: [PATCH 11/23] docs: update .env.example for mongodb connection string - Changes the example DATABASE_URL to the standard MongoDB format. --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index e7382eb..1f4422a 100644 --- a/.env.example +++ b/.env.example @@ -2,4 +2,4 @@ # Copy this file to .env and fill in your actual configuration values. # The .env file is ignored by Git and should NOT be committed. -DATABASE_URL="postgres://user:password@localhost:5432/db_name" \ No newline at end of file +DATABASE_URL="mongodb://user:password@localhost:27017/ht_api_db" \ No newline at end of file From b07d6d2b7a220d33ff30286c92f1ccbe43cabd31 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 14:22:10 +0100 Subject: [PATCH 12/23] fix(config): make .env file loading robust - Refactors `EnvironmentConfig` to actively search for the `.env` file by traversing up the directory tree from the current working directory. - This change makes the server startup process resilient to cases where the Dart Frog development server's execution context is not the project root, resolving the "file not found" error for `.env`. --- .env | 6 ++++ lib/src/config/environment_config.dart | 47 +++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..86770c0 --- /dev/null +++ b/.env @@ -0,0 +1,6 @@ +# This is an example environment file. +# Copy this file to .env and fill in your actual configuration values. +# The .env file is ignored by Git and should NOT be committed. + +# DATABASE_URL="mongodb://user:password@localhost:27017/ht_api_db" +DATABASE_URL="mongodb://localhost:27017/ht_api_db" diff --git a/lib/src/config/environment_config.dart b/lib/src/config/environment_config.dart index 0c6eeee..8967dbe 100644 --- a/lib/src/config/environment_config.dart +++ b/lib/src/config/environment_config.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + import 'package:dotenv/dotenv.dart'; import 'package:logging/logging.dart'; @@ -11,11 +13,46 @@ import 'package:logging/logging.dart'; abstract final class EnvironmentConfig { static final _log = Logger('EnvironmentConfig'); - // The DotEnv instance that loads the .env file and platform variables. - // It's initialized once and reused. - static final _env = DotEnv(includePlatformEnvironment: true)..load(); + // The DotEnv instance is now loaded via a helper method to make it more + // resilient to current working directory issues. + static final _env = _loadEnv(); + + /// Helper method to load the .env file more robustly. + /// + /// It searches for the .env file starting from the current directory + /// and moving up to parent directories. This makes it resilient to + /// issues where the execution context's working directory is not the + /// project root. + static DotEnv _loadEnv() { + final env = DotEnv(includePlatformEnvironment: true); + var dir = Directory.current; + + // Loop to search up the directory tree for the .env file. + while (true) { + final envFile = File('${dir.path}/.env'); + if (envFile.existsSync()) { + _log.info('Found .env file at: ${envFile.path}'); + // Load the variables from the found file. + env.load([envFile.path]); + return env; + } + + // Stop if we have reached the root of the filesystem. + if (dir.parent.path == dir.path) { + _log.warning( + '.env file not found by searching. Falling back to default load().', + ); + // Fallback to the original behavior if no file is found. + env.load(); + return env; + } + + // Move up to the parent directory. + dir = dir.parent; + } + } - /// Retrieves the PostgreSQL database connection URI from the environment. + /// Retrieves the database connection URI from the environment. /// /// The value is read from the `DATABASE_URL` environment variable. /// @@ -33,7 +70,7 @@ abstract final class EnvironmentConfig { return dbUrl; } - /// Retrieves the current environment mode (e.g., 'development', 'production'). + /// Retrieves the current environment mode (e.g., 'development'). /// /// The value is read from the `ENV` environment variable. /// Defaults to 'production' if the variable is not set. From 7b24b535df643aafa90bebce3f1ab91440abc2d1 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 14:26:31 +0100 Subject: [PATCH 13/23] /// issues where the execution context's working directory is not the /// project root. static DotEnv _loadEnv() { final env = DotEnv(includePlatformEnvironment: true); try { // Find the project root by looking for pubspec.yaml, then find .env var dir = Directory.current; while (true) { final pubspecFile = File('${dir.path}/pubspec.yaml'); if (pubspecFile.existsSync()) { // Found project root, now look for .env in this directory final envFile = File('${dir.path}/.env'); if (envFile.existsSync()) { _log.info('Found .env file at: ${envFile.path}'); env.load([envFile.path]); return env; } break; // Found pubspec but no .env, break and fall back } // Stop if we have reached the root of the filesystem. if (dir.parent.path == dir.path) { break; } dir = dir.parent; } } catch (e) { _log.warning('Error during robust .env search: $e. Falling back.'); } // Fallback for when the robust search fails _log.warning( '.env file not found by searching for project root. ' 'Falling back to default load().', ); env.load(); return env; } /// Retrieves the database connection URI from the environment. --- lib/src/config/environment_config.dart | 52 +++++++++++++++----------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/lib/src/config/environment_config.dart b/lib/src/config/environment_config.dart index 8967dbe..a034c2c 100644 --- a/lib/src/config/environment_config.dart +++ b/lib/src/config/environment_config.dart @@ -25,31 +25,39 @@ abstract final class EnvironmentConfig { /// project root. static DotEnv _loadEnv() { final env = DotEnv(includePlatformEnvironment: true); - var dir = Directory.current; + try { + // Find the project root by looking for pubspec.yaml, then find .env + var dir = Directory.current; + while (true) { + final pubspecFile = File('${dir.path}/pubspec.yaml'); + if (pubspecFile.existsSync()) { + // Found project root, now look for .env in this directory + final envFile = File('${dir.path}/.env'); + if (envFile.existsSync()) { + _log.info('Found .env file at: ${envFile.path}'); + env.load([envFile.path]); + return env; + } + break; // Found pubspec but no .env, break and fall back + } - // Loop to search up the directory tree for the .env file. - while (true) { - final envFile = File('${dir.path}/.env'); - if (envFile.existsSync()) { - _log.info('Found .env file at: ${envFile.path}'); - // Load the variables from the found file. - env.load([envFile.path]); - return env; + // Stop if we have reached the root of the filesystem. + if (dir.parent.path == dir.path) { + break; + } + dir = dir.parent; } - - // Stop if we have reached the root of the filesystem. - if (dir.parent.path == dir.path) { - _log.warning( - '.env file not found by searching. Falling back to default load().', - ); - // Fallback to the original behavior if no file is found. - env.load(); - return env; - } - - // Move up to the parent directory. - dir = dir.parent; + } catch (e) { + _log.warning('Error during robust .env search: $e. Falling back.'); } + + // Fallback for when the robust search fails + _log.warning( + '.env file not found by searching for project root. ' + 'Falling back to default load().', + ); + env.load(); + return env; } /// Retrieves the database connection URI from the environment. From 20a92d220c72c8e109a3b6f5249f652eac12c57a Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 14:32:10 +0100 Subject: [PATCH 14/23] fix(config): correct directory traversal in .env search - Fixes a subtle error in the directory traversal logic within the `EnvironmentConfig._loadEnv` method. The previous implementation had a flaw in how it checked for the parent directory, which could prevent it from correctly locating the `pubspec.yaml` and `.env` files in certain project structures or when the Dart Frog development server starts from a subdirectory. - The corrected logic now accurately traverses up the directory tree, repeatedly checking if the parent directory is the same as the current directory (indicating the root of the filesystem). This ensures that the search correctly identifies the project root and locates the `.env` file. --- lib/src/config/environment_config.dart | 55 +++++++++++--------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/lib/src/config/environment_config.dart b/lib/src/config/environment_config.dart index a034c2c..43cbd43 100644 --- a/lib/src/config/environment_config.dart +++ b/lib/src/config/environment_config.dart @@ -24,40 +24,33 @@ abstract final class EnvironmentConfig { /// issues where the execution context's working directory is not the /// project root. static DotEnv _loadEnv() { - final env = DotEnv(includePlatformEnvironment: true); - try { - // Find the project root by looking for pubspec.yaml, then find .env - var dir = Directory.current; - while (true) { - final pubspecFile = File('${dir.path}/pubspec.yaml'); - if (pubspecFile.existsSync()) { - // Found project root, now look for .env in this directory - final envFile = File('${dir.path}/.env'); - if (envFile.existsSync()) { - _log.info('Found .env file at: ${envFile.path}'); - env.load([envFile.path]); - return env; - } - break; // Found pubspec but no .env, break and fall back + final env = DotEnv(includePlatformEnvironment: true); // Start with default + var currentDir = Directory.current; + _log.fine('Starting .env search from: ${currentDir.path}'); + // Traverse up the directory tree to find pubspec.yaml + while (currentDir.parent.path != currentDir.path) { + final pubspecPath = '${currentDir.path}/pubspec.yaml'; + _log.finer('Checking for pubspec.yaml at: '); + if (File(pubspecPath).existsSync()) { + // Found pubspec.yaml, now load .env from the same directory + final envPath = '${currentDir.path}/.env'; + _log.info('Found pubspec.yaml, now looking for .env at: '); + if (File(envPath).existsSync()) { + _log.info('Found .env file at: '); + env.load([envPath]); // Load variables from the found .env file + return env; // Return immediately upon finding + } else { + _log.warning('pubspec.yaml found, but no .env in the same directory.'); + break; // Stop searching since pubspec.yaml should contain .env } - - // Stop if we have reached the root of the filesystem. - if (dir.parent.path == dir.path) { - break; - } - dir = dir.parent; } - } catch (e) { - _log.warning('Error during robust .env search: $e. Falling back.'); + currentDir = currentDir.parent; // Move to the parent directory + _log.finer('Moving up to parent directory: ${currentDir.path}'); } - - // Fallback for when the robust search fails - _log.warning( - '.env file not found by searching for project root. ' - 'Falling back to default load().', - ); - env.load(); - return env; + // If loop completes without returning, .env was not found + _log.warning('.env not found by searching. Falling back to default load().'); + env.load(); // Fallback to default load + return env; // Return even if fallback } /// Retrieves the database connection URI from the environment. From 96912c2f29e3b3f46ab1531a9a13300739ac1da3 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 18:22:35 +0100 Subject: [PATCH 15/23] feat(config): enhance .env loading logs --- lib/src/config/environment_config.dart | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/src/config/environment_config.dart b/lib/src/config/environment_config.dart index 43cbd43..fd8e18a 100644 --- a/lib/src/config/environment_config.dart +++ b/lib/src/config/environment_config.dart @@ -29,14 +29,17 @@ abstract final class EnvironmentConfig { _log.fine('Starting .env search from: ${currentDir.path}'); // Traverse up the directory tree to find pubspec.yaml while (currentDir.parent.path != currentDir.path) { - final pubspecPath = '${currentDir.path}/pubspec.yaml'; - _log.finer('Checking for pubspec.yaml at: '); + final pubspecPath = + '${currentDir.path}${Platform.pathSeparator}pubspec.yaml'; + _log.finer('Checking for pubspec.yaml at: $pubspecPath'); if (File(pubspecPath).existsSync()) { // Found pubspec.yaml, now load .env from the same directory - final envPath = '${currentDir.path}/.env'; - _log.info('Found pubspec.yaml, now looking for .env at: '); + final envPath = '${currentDir.path}${Platform.pathSeparator}.env'; + _log.info( + 'Found pubspec.yaml, now looking for .env at: ${currentDir.path}', + ); if (File(envPath).existsSync()) { - _log.info('Found .env file at: '); + _log.info('Found .env file at: $envPath'); env.load([envPath]); // Load variables from the found .env file return env; // Return immediately upon finding } else { From e7d0d8d52ccb822ac655e403df5b158c5fe82335 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 18:24:43 +0100 Subject: [PATCH 16/23] fix(config): prevent state corruption on failed dependency init --- lib/src/config/app_dependencies.dart | 259 ++++++++++++++------------- 1 file changed, 137 insertions(+), 122 deletions(-) diff --git a/lib/src/config/app_dependencies.dart b/lib/src/config/app_dependencies.dart index 0342396..22123e6 100644 --- a/lib/src/config/app_dependencies.dart +++ b/lib/src/config/app_dependencies.dart @@ -32,6 +32,7 @@ class AppDependencies { static AppDependencies get instance => _instance; bool _isInitialized = false; + Exception? _initializationError; final _log = Logger('AppDependencies'); // --- Late-initialized fields for all dependencies --- @@ -64,132 +65,146 @@ class AppDependencies { /// /// This method is idempotent; it will only run the initialization logic once. Future init() async { + // If initialization previously failed, re-throw the original error. + if (_initializationError != null) { + throw _initializationError!; + } + if (_isInitialized) return; _log.info('Initializing application dependencies...'); - // 1. Initialize Database Connection - _mongoDbConnectionManager = MongoDbConnectionManager(); - await _mongoDbConnectionManager.init(EnvironmentConfig.databaseUrl); - _log.info('MongoDB connection established.'); - - // 2. Seed Database - final seedingService = DatabaseSeedingService( - db: _mongoDbConnectionManager.db, - log: Logger('DatabaseSeedingService'), - ); - await seedingService.seedInitialData(); - _log.info('Database seeding complete.'); - - // 3. Initialize Data Clients (MongoDB implementation) - final headlineClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'headlines', - fromJson: Headline.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final topicClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'topics', - fromJson: Topic.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final sourceClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'sources', - fromJson: Source.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final countryClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'countries', - fromJson: Country.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final userClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'users', - fromJson: User.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final userAppSettingsClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'user_app_settings', - fromJson: UserAppSettings.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final userContentPreferencesClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'user_content_preferences', - fromJson: UserContentPreferences.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - final remoteConfigClient = HtDataMongodb( - connectionManager: _mongoDbConnectionManager, - modelName: 'remote_configs', - fromJson: RemoteConfig.fromJson, - toJson: (item) => item.toJson(), - logger: Logger('HtDataMongodb'), - ); - - // 4. Initialize Repositories - headlineRepository = HtDataRepository(dataClient: headlineClient); - topicRepository = HtDataRepository(dataClient: topicClient); - sourceRepository = HtDataRepository(dataClient: sourceClient); - countryRepository = HtDataRepository(dataClient: countryClient); - userRepository = HtDataRepository(dataClient: userClient); - userAppSettingsRepository = - HtDataRepository(dataClient: userAppSettingsClient); - userContentPreferencesRepository = - HtDataRepository(dataClient: userContentPreferencesClient); - remoteConfigRepository = HtDataRepository(dataClient: remoteConfigClient); - - final emailClient = HtEmailInMemoryClient( - logger: Logger('HtEmailInMemoryClient'), - ); - emailRepository = HtEmailRepository(emailClient: emailClient); - - // 5. Initialize Services - tokenBlacklistService = InMemoryTokenBlacklistService( - log: Logger('InMemoryTokenBlacklistService'), - ); - authTokenService = JwtAuthTokenService( - userRepository: userRepository, - blacklistService: tokenBlacklistService, - uuidGenerator: const Uuid(), - log: Logger('JwtAuthTokenService'), - ); - verificationCodeStorageService = InMemoryVerificationCodeStorageService(); - authService = AuthService( - userRepository: userRepository, - authTokenService: authTokenService, - verificationCodeStorageService: verificationCodeStorageService, - emailRepository: emailRepository, - userAppSettingsRepository: userAppSettingsRepository, - userContentPreferencesRepository: userContentPreferencesRepository, - uuidGenerator: const Uuid(), - log: Logger('AuthService'), - ); - dashboardSummaryService = DashboardSummaryService( - headlineRepository: headlineRepository, - topicRepository: topicRepository, - sourceRepository: sourceRepository, - ); - permissionService = const PermissionService(); - userPreferenceLimitService = DefaultUserPreferenceLimitService( - remoteConfigRepository: remoteConfigRepository, - log: Logger('DefaultUserPreferenceLimitService'), - ); - - _isInitialized = true; - _log.info('Application dependencies initialized successfully.'); + try { + // 1. Initialize Database Connection + _mongoDbConnectionManager = MongoDbConnectionManager(); + await _mongoDbConnectionManager.init(EnvironmentConfig.databaseUrl); + _log.info('MongoDB connection established.'); + + // 2. Seed Database + final seedingService = DatabaseSeedingService( + db: _mongoDbConnectionManager.db, + log: Logger('DatabaseSeedingService'), + ); + await seedingService.seedInitialData(); + _log.info('Database seeding complete.'); + + // 3. Initialize Data Clients (MongoDB implementation) + final headlineClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'headlines', + fromJson: Headline.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final topicClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'topics', + fromJson: Topic.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final sourceClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'sources', + fromJson: Source.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final countryClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'countries', + fromJson: Country.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final userClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'users', + fromJson: User.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final userAppSettingsClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'user_app_settings', + fromJson: UserAppSettings.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final userContentPreferencesClient = + HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'user_content_preferences', + fromJson: UserContentPreferences.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + final remoteConfigClient = HtDataMongodb( + connectionManager: _mongoDbConnectionManager, + modelName: 'remote_configs', + fromJson: RemoteConfig.fromJson, + toJson: (item) => item.toJson(), + logger: Logger('HtDataMongodb'), + ); + + // 4. Initialize Repositories + headlineRepository = HtDataRepository(dataClient: headlineClient); + topicRepository = HtDataRepository(dataClient: topicClient); + sourceRepository = HtDataRepository(dataClient: sourceClient); + countryRepository = HtDataRepository(dataClient: countryClient); + userRepository = HtDataRepository(dataClient: userClient); + userAppSettingsRepository = + HtDataRepository(dataClient: userAppSettingsClient); + userContentPreferencesRepository = + HtDataRepository(dataClient: userContentPreferencesClient); + remoteConfigRepository = + HtDataRepository(dataClient: remoteConfigClient); + + final emailClient = HtEmailInMemoryClient( + logger: Logger('HtEmailInMemoryClient'), + ); + emailRepository = HtEmailRepository(emailClient: emailClient); + + // 5. Initialize Services + tokenBlacklistService = InMemoryTokenBlacklistService( + log: Logger('InMemoryTokenBlacklistService'), + ); + authTokenService = JwtAuthTokenService( + userRepository: userRepository, + blacklistService: tokenBlacklistService, + uuidGenerator: const Uuid(), + log: Logger('JwtAuthTokenService'), + ); + verificationCodeStorageService = + InMemoryVerificationCodeStorageService(); + authService = AuthService( + userRepository: userRepository, + authTokenService: authTokenService, + verificationCodeStorageService: verificationCodeStorageService, + emailRepository: emailRepository, + userAppSettingsRepository: userAppSettingsRepository, + userContentPreferencesRepository: userContentPreferencesRepository, + uuidGenerator: const Uuid(), + log: Logger('AuthService'), + ); + dashboardSummaryService = DashboardSummaryService( + headlineRepository: headlineRepository, + topicRepository: topicRepository, + sourceRepository: sourceRepository, + ); + permissionService = const PermissionService(); + userPreferenceLimitService = DefaultUserPreferenceLimitService( + remoteConfigRepository: remoteConfigRepository, + log: Logger('DefaultUserPreferenceLimitService'), + ); + + _isInitialized = true; + _log.info('Application dependencies initialized successfully.'); + } on Exception catch (e) { + _log.severe('Failed to initialize application dependencies', e); + _initializationError = e; + rethrow; + } } /// Disposes of resources, such as closing the database connection. From d1ac01a256c6453040d87eb14e33bfa8b683b022 Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 18:30:15 +0100 Subject: [PATCH 17/23] fix(config): make dependency init robust against all throwables --- lib/src/config/app_dependencies.dart | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/src/config/app_dependencies.dart b/lib/src/config/app_dependencies.dart index 22123e6..2ae1b7e 100644 --- a/lib/src/config/app_dependencies.dart +++ b/lib/src/config/app_dependencies.dart @@ -32,7 +32,8 @@ class AppDependencies { static AppDependencies get instance => _instance; bool _isInitialized = false; - Exception? _initializationError; + Object? _initializationError; + StackTrace? _initializationStackTrace; final _log = Logger('AppDependencies'); // --- Late-initialized fields for all dependencies --- @@ -67,7 +68,7 @@ class AppDependencies { Future init() async { // If initialization previously failed, re-throw the original error. if (_initializationError != null) { - throw _initializationError!; + return Future.error(_initializationError!, _initializationStackTrace); } if (_isInitialized) return; @@ -200,9 +201,10 @@ class AppDependencies { _isInitialized = true; _log.info('Application dependencies initialized successfully.'); - } on Exception catch (e) { - _log.severe('Failed to initialize application dependencies', e); + } catch (e, s) { + _log.severe('Failed to initialize application dependencies', e, s); _initializationError = e; + _initializationStackTrace = s; rethrow; } } From 6e84bcf068fb15c78471baf255866b6bcee7d5be Mon Sep 17 00:00:00 2001 From: fulleni Date: Fri, 11 Jul 2025 19:31:52 +0100 Subject: [PATCH 18/23] chore --- lib/src/services/database_seeding_service.dart | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/lib/src/services/database_seeding_service.dart b/lib/src/services/database_seeding_service.dart index fbd0103..76de4ee 100644 --- a/lib/src/services/database_seeding_service.dart +++ b/lib/src/services/database_seeding_service.dart @@ -80,30 +80,19 @@ class DatabaseSeedingService { final operations = >[]; for (final item in fixtureData) { - final id = getId(item); - - if (!ObjectId.isValidHexId(id)) { - _log.warning('Skipping document with invalid ID format: $id'); - continue; - } - - final objectId = ObjectId.fromHexString(id); + // Generate a new ObjectId for each document + final objectId = ObjectId(); final document = toJson(item)..remove('id'); operations.add({ 'replaceOne': { - 'filter': {'_id': objectId}, + 'filter': {}, // Match all documents (replace existing or insert new) 'replacement': document, 'upsert': true, }, }); } - if (operations.isEmpty) { - _log.info('No valid documents to write for "$collectionName".'); - return; - } - final result = await collection.bulkWrite(operations); _log.info( 'Seeding for "$collectionName" complete. ' From 4ce3cc5203d0a1e6d1516a3934a4129b06ce4a0e Mon Sep 17 00:00:00 2001 From: fulleni Date: Sat, 12 Jul 2025 06:45:53 +0100 Subject: [PATCH 19/23] fix(api): make database seeding idempotent and preserve IDs Corrects the `DatabaseSeedingService` to ensure the seeding process is idempotent and correctly preserves the relationships between fixture documents. The previous implementation had two critical bugs: 1. It generated a new, random `ObjectId` on every run, which broke all predefined relationships between entities. 2. It used an empty filter `{}` in its `replaceOne` operation, causing it to overwrite existing documents incorrectly on subsequent runs. This fix addresses these issues by: - Using the predefined hex string ID from each fixture item to create a deterministic `ObjectId`. - Using `updateOne` with a specific `_id` in the filter, ensuring that the correct document is targeted. - Using the `$set` operator with `upsert: true` to safely insert or update the document, making the entire process idempotent. --- lib/src/services/database_seeding_service.dart | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/src/services/database_seeding_service.dart b/lib/src/services/database_seeding_service.dart index 76de4ee..86df5bf 100644 --- a/lib/src/services/database_seeding_service.dart +++ b/lib/src/services/database_seeding_service.dart @@ -80,14 +80,19 @@ class DatabaseSeedingService { final operations = >[]; for (final item in fixtureData) { - // Generate a new ObjectId for each document - final objectId = ObjectId(); + // Use the predefined hex string ID from the fixture to create a + // deterministic ObjectId. This is crucial for maintaining relationships + // between documents (e.g., a headline and its source). + final objectId = ObjectId.fromHexString(getId(item)); final document = toJson(item)..remove('id'); operations.add({ - 'replaceOne': { - 'filter': {}, // Match all documents (replace existing or insert new) - 'replacement': document, + // Use updateOne with $set to be less destructive than replaceOne. + 'updateOne': { + // Filter by the specific, deterministic _id. + 'filter': {'_id': objectId}, + // Set the fields of the document. + 'update': {'\$set': document}, 'upsert': true, }, }); From 520747545eab169cc91261144133c4e62b71c776 Mon Sep 17 00:00:00 2001 From: fulleni Date: Sat, 12 Jul 2025 06:48:25 +0100 Subject: [PATCH 20/23] fix(api): ensure all user data is deleted on account deletion Fixes a critical bug in the `AuthService.deleteAccount` method where only the main user document was being deleted. The previous logic incorrectly assumed a relational `CASCADE` delete behavior. This change adds explicit calls to delete the user's associated documents from the `user_app_settings` and `user_content_preferences` collections, ensuring that no orphaned data is left in the database after an account is deleted. --- lib/src/services/auth_service.dart | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/src/services/auth_service.dart b/lib/src/services/auth_service.dart index b015d61..dc38237 100644 --- a/lib/src/services/auth_service.dart +++ b/lib/src/services/auth_service.dart @@ -549,25 +549,29 @@ class AuthService { /// Throws [NotFoundException] if the user does not exist. /// Throws [OperationFailedException] for other errors during deletion or cleanup. Future deleteAccount({required String userId}) async { - // Note: The user record itself is deleted via a CASCADE constraint - // when the corresponding entry in the `users` table is deleted. - // This is because `user_app_settings.user_id` and - // `user_content_preferences.user_id` have `ON DELETE CASCADE`. - // Therefore, we only need to delete the main user record. try { // Fetch the user first to get their email if needed for cleanup final userToDelete = await _userRepository.read(id: userId); _log.info('Found user ${userToDelete.id} for deletion.'); - // 1. Delete the main user record from the `users` table. - // The `ON DELETE CASCADE` constraint on the `user_app_settings` and - // `user_content_preferences` tables will automatically delete the - // associated records in those tables. This also implicitly invalidates + // 1. Explicitly delete associated user data. Unlike relational databases + // with CASCADE constraints, MongoDB requires manual deletion of related + // documents in different collections. + await _userAppSettingsRepository.delete(id: userId, userId: userId); + _log.info('Deleted UserAppSettings for user ${userToDelete.id}.'); + + await _userContentPreferencesRepository.delete( + id: userId, + userId: userId, + ); + _log.info('Deleted UserContentPreferences for user ${userToDelete.id}.'); + + // 2. Delete the main user record. This also implicitly invalidates // tokens that rely on user lookup, as the user will no longer exist. await _userRepository.delete(id: userId); _log.info('User ${userToDelete.id} deleted from repository.'); - // 2. Clear any pending verification codes for this user ID (linking). + // 3. Clear any pending verification codes for this user ID (linking). try { await _verificationCodeStorageService.clearLinkCode(userId); _log.info('Cleared link code for user ${userToDelete.id}.'); @@ -578,7 +582,7 @@ class AuthService { ); } - // 3. Clear any pending sign-in codes for the user's email (if they had one). + // 4. Clear any pending sign-in codes for the user's email (if they had one). // The email for anonymous users is a placeholder and not used for sign-in. if (userToDelete.appRole != AppUserRole.guestUser) { try { From dbae817d8b128d72bf50fa0d59aff595381e5f90 Mon Sep 17 00:00:00 2001 From: fulleni Date: Sat, 12 Jul 2025 08:01:48 +0100 Subject: [PATCH 21/23] refactor(api): use repository count method for dashboard summary Replaces the `readAll()` calls in the `DashboardSummaryService` with the more efficient `count()` method from the data repository. This avoids fetching all documents from the database just to get their count, significantly improving the performance of the dashboard summary endpoint by leveraging the native counting capabilities of the database. --- .../services/dashboard_summary_service.dart | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/src/services/dashboard_summary_service.dart b/lib/src/services/dashboard_summary_service.dart index ba9bfdc..478a281 100644 --- a/lib/src/services/dashboard_summary_service.dart +++ b/lib/src/services/dashboard_summary_service.dart @@ -23,26 +23,26 @@ class DashboardSummaryService { /// Calculates and returns the current dashboard summary. /// - /// This method fetches all items from the required repositories to count them - /// and constructs a [DashboardSummary] object. + /// This method fetches the counts of all items from the required + /// repositories and constructs a [DashboardSummary] object. Future getSummary() async { // Use Future.wait to fetch all counts in parallel for efficiency. final results = await Future.wait([ - _headlineRepository.readAll(), - _topicRepository.readAll(), - _sourceRepository.readAll(), + _headlineRepository.count(), + _topicRepository.count(), + _sourceRepository.count(), ]); - // The results are PaginatedResponse objects. - final headlineResponse = results[0] as PaginatedResponse; - final topicResponse = results[1] as PaginatedResponse; - final sourceResponse = results[2] as PaginatedResponse; + // The results are integers. + final headlineCount = results[0]; + final topicCount = results[1]; + final sourceCount = results[2]; return DashboardSummary( id: 'dashboard_summary', // Fixed ID for the singleton summary - headlineCount: headlineResponse.items.length, - topicCount: topicResponse.items.length, - sourceCount: sourceResponse.items.length, + headlineCount: headlineCount, + topicCount: topicCount, + sourceCount: sourceCount, ); } } From c49d2c46cbf988ded19679f40f9d2ffe91f850a7 Mon Sep 17 00:00:00 2001 From: fulleni Date: Sat, 12 Jul 2025 08:09:59 +0100 Subject: [PATCH 22/23] fix(api): improve error handling for invalid JSON body Adds a specific error handler for `CheckedFromJsonException` to return a 400 Bad Request instead of a generic 500 Internal Server Error when deserialization fails due to invalid client input. This provides more accurate feedback to API clients when they send a request body that is missing required fields or contains incorrect data types. --- lib/src/middlewares/error_handler.dart | 16 ++++++++++++++++ pubspec.yaml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/src/middlewares/error_handler.dart b/lib/src/middlewares/error_handler.dart index 2778590..a51fcee 100644 --- a/lib/src/middlewares/error_handler.dart +++ b/lib/src/middlewares/error_handler.dart @@ -5,6 +5,7 @@ import 'dart:io'; import 'package:dart_frog/dart_frog.dart'; import 'package:ht_shared/ht_shared.dart'; +import 'package:json_annotation/json_annotation.dart'; /// Middleware that catches errors and converts them into /// standardized JSON responses. @@ -26,6 +27,21 @@ Middleware errorHandler() { 'error': {'code': errorCode, 'message': e.message}, }, ); + } on CheckedFromJsonException catch (e, stackTrace) { + // Handle json_serializable validation errors. These are client errors. + final field = e.key ?? 'unknown'; + final message = 'Invalid request body: Field "$field" has an ' + 'invalid value or is missing. ${e.message}'; + print('CheckedFromJsonException Caught: $e\n$stackTrace'); + return Response.json( + statusCode: HttpStatus.badRequest, // 400 + body: { + 'error': { + 'code': 'invalidField', + 'message': message, + }, + }, + ); } on FormatException catch (e, stackTrace) { // Handle data format/parsing errors (often indicates bad client input) print('FormatException Caught: $e\n$stackTrace'); // Log for debugging diff --git a/pubspec.yaml b/pubspec.yaml index fb6e8d2..96e31e9 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -34,7 +34,7 @@ dependencies: ht_shared: git: url: https://github.com/headlines-toolkit/ht-shared.git - + json_annotation: ^4.9.0 logging: ^1.3.0 meta: ^1.16.0 mongo_dart: ^0.10.5 From b2b43634b193851e04701fad9115cb80a831c3e9 Mon Sep 17 00:00:00 2001 From: fulleni Date: Sat, 12 Jul 2025 08:22:12 +0100 Subject: [PATCH 23/23] fix(api): correct admin data scoping in generic data handlers Fixes a bug where administrators were incorrectly scoped to their own userId when accessing user-owned resources, preventing them from managing other users' data. The logic in the generic data handlers (`/data` and `/data/[id]`) has been updated to only apply the `userId` filter to repository calls if the model is user-owned AND the authenticated user is not an admin. This allows administrators to perform global operations as intended. --- routes/api/v1/data/[id]/index.dart | 9 ++++++--- routes/api/v1/data/index.dart | 13 +++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/routes/api/v1/data/[id]/index.dart b/routes/api/v1/data/[id]/index.dart index e0cd998..2190367 100644 --- a/routes/api/v1/data/[id]/index.dart +++ b/routes/api/v1/data/[id]/index.dart @@ -87,7 +87,8 @@ Future _handleGet( // for filtering. Otherwise, pass null. // Note: This is for data *scoping* by the repository, not the permission check. // We infer user-owned based on the presence of getOwnerId function. - if (modelConfig.getOwnerId != null) { + if (modelConfig.getOwnerId != null && + !permissionService.isAdmin(authenticatedUser)) { userIdForRepoCall = authenticatedUser.id; } else { userIdForRepoCall = null; @@ -276,7 +277,8 @@ Future _handlePut( String? userIdForRepoCall; // If the model is user-owned, pass the authenticated user's ID to the repository // for ownership enforcement. Otherwise, pass null. - if (modelConfig.getOwnerId != null) { + if (modelConfig.getOwnerId != null && + !permissionService.isAdmin(authenticatedUser)) { userIdForRepoCall = authenticatedUser.id; } else { userIdForRepoCall = null; @@ -445,7 +447,8 @@ Future _handleDelete( String? userIdForRepoCall; // If the model is user-owned, pass the authenticated user's ID to the repository // for ownership enforcement. Otherwise, pass null. - if (modelConfig.getOwnerId != null) { + if (modelConfig.getOwnerId != null && + !permissionService.isAdmin(authenticatedUser)) { userIdForRepoCall = authenticatedUser.id; } else { userIdForRepoCall = null; diff --git a/routes/api/v1/data/index.dart b/routes/api/v1/data/index.dart index 01bfdd9..82ed8d8 100644 --- a/routes/api/v1/data/index.dart +++ b/routes/api/v1/data/index.dart @@ -4,6 +4,7 @@ import 'dart:io'; import 'package:dart_frog/dart_frog.dart'; import 'package:ht_api/src/registry/model_registry.dart'; import 'package:ht_data_repository/ht_data_repository.dart'; +import 'package:ht_api/src/rbac/permission_service.dart'; import 'package:ht_shared/ht_shared.dart'; import '../../../_middleware.dart'; // For RequestId @@ -77,8 +78,10 @@ Future _handleGet(RequestContext context) async { } // --- Repository Call --- - final userIdForRepoCall = - (modelConfig.getOwnerId != null) ? authenticatedUser.id : null; + final userIdForRepoCall = (modelConfig.getOwnerId != null && + !context.read().isAdmin(authenticatedUser)) + ? authenticatedUser.id + : null; dynamic responseData; @@ -175,8 +178,10 @@ Future _handlePost(RequestContext context) async { } // --- Repository Call --- - final userIdForRepoCall = - (modelConfig.getOwnerId != null) ? authenticatedUser.id : null; + final userIdForRepoCall = (modelConfig.getOwnerId != null && + !context.read().isAdmin(authenticatedUser)) + ? authenticatedUser.id + : null; dynamic createdItem; switch (modelName) {