diff --git a/app/build.yaml b/app/build.yaml index 2167c8adca..36e2c18c14 100644 --- a/app/build.yaml +++ b/app/build.yaml @@ -8,6 +8,7 @@ targets: - 'lib/account/models.dart' - 'lib/admin/models.dart' - 'lib/dartdoc/models.dart' + - 'lib/database/*.dart' - 'lib/frontend/handlers/pubapi.dart' - 'lib/frontend/handlers/routes.dart' - 'lib/frontend/handlers/redirects.dart' diff --git a/app/lib/database/database.dart b/app/lib/database/database.dart index 9f194de3ea..f8022ee615 100644 --- a/app/lib/database/database.dart +++ b/app/lib/database/database.dart @@ -10,9 +10,11 @@ import 'package:clock/clock.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:meta/meta.dart'; import 'package:postgres/postgres.dart'; +import 'package:pub_dev/database/schema.dart'; import 'package:pub_dev/service/secret/backend.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/env_config.dart'; +import 'package:typed_sql/typed_sql.dart'; final _random = Random.secure(); @@ -27,8 +29,10 @@ PrimaryDatabase? get primaryDatabase => /// Access to the primary database connection and object mapping. class PrimaryDatabase { final Pool _pg; + final DatabaseAdapter _adapter; + final Database db; - PrimaryDatabase._(this._pg); + PrimaryDatabase._(this._pg, this._adapter, this.db); /// Gets the connection string either from the environment variable or from /// the secret backend, connects to it and registers the primary database @@ -77,10 +81,14 @@ class PrimaryDatabase { static Future _fromConnectionString(String value) async { final pg = Pool.withUrl(value); - return PrimaryDatabase._(pg); + final adapter = DatabaseAdapter.postgres(pg); + final db = Database(adapter, SqlDialect.postgres()); + await db.createTables(); + return PrimaryDatabase._(pg, adapter, db); } Future close() async { + await _adapter.close(); await _pg.close(); } diff --git a/app/lib/database/schema.dart b/app/lib/database/schema.dart new file mode 100644 index 0000000000..ce9e750467 --- /dev/null +++ b/app/lib/database/schema.dart @@ -0,0 +1,20 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'package:json_annotation/json_annotation.dart'; +import 'package:meta/meta.dart'; +import 'package:pub_dev/admin/actions/actions.dart'; +import 'package:pub_dev/task/models.dart'; +import 'package:typed_sql/typed_sql.dart'; + +part 'schema.g.dart'; +part 'schema.task.dart'; + +abstract final class PrimarySchema extends Schema { + Table get tasks; + + Table get taskDependencies; +} diff --git a/app/lib/database/schema.g.dart b/app/lib/database/schema.g.dart new file mode 100644 index 0000000000..0ef49ff026 --- /dev/null +++ b/app/lib/database/schema.g.dart @@ -0,0 +1,923 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'schema.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +TaskState _$TaskStateFromJson(Map json) => TaskState( + versions: (json['versions'] as Map).map( + (k, e) => MapEntry( + k, + PackageVersionStateInfo.fromJson(e as Map), + ), + ), + abortedTokens: (json['abortedTokens'] as List) + .map((e) => AbortedTokenInfo.fromJson(e as Map)) + .toList(), +); + +Map _$TaskStateToJson(TaskState instance) => { + 'versions': instance.versions, + 'abortedTokens': instance.abortedTokens, +}; + +// ************************************************************************** +// Generator: _TypedSqlBuilder +// ************************************************************************** + +/// Extension methods for a [Database] operating on [PrimarySchema]. +extension PrimarySchemaSchema on Database { + static const _$tables = [_$Task._$table, _$TaskDependency._$table]; + + Table get tasks => ExposedForCodeGen.declareTable(this, _$Task._$table); + + Table get taskDependencies => + ExposedForCodeGen.declareTable(this, _$TaskDependency._$table); + + /// Create tables defined in [PrimarySchema]. + /// + /// Calling this on an empty database will create the tables + /// defined in [PrimarySchema]. In production it's often better to + /// use [createPrimarySchemaTables] and manage migrations using + /// external tools. + /// + /// This method is mostly useful for testing. + /// + /// > [!WARNING] + /// > If the database is **not empty** behavior is undefined, most + /// > likely this operation will fail. + Future createTables() async => + ExposedForCodeGen.createTables(context: this, tables: _$tables); +} + +/// Get SQL [DDL statements][1] for tables defined in [PrimarySchema]. +/// +/// This returns a SQL script with multiple DDL statements separated by `;` +/// using the specified [dialect]. +/// +/// Executing these statements in an empty database will create the tables +/// defined in [PrimarySchema]. In practice, this method is often used for +/// printing the DDL statements, such that migrations can be managed by +/// external tools. +/// +/// [1]: https://en.wikipedia.org/wiki/Data_definition_language +String createPrimarySchemaTables(SqlDialect dialect) => + ExposedForCodeGen.createTableSchema( + dialect: dialect, + tables: PrimarySchemaSchema._$tables, + ); + +final class _$Task extends Task { + _$Task._( + this.runtimeVersion, + this.package, + this.state, + this.pendingAt, + this.lastDependencyChanged, + this.finished, + ); + + @override + final String runtimeVersion; + + @override + final String package; + + @override + final TaskState state; + + @override + final DateTime pendingAt; + + @override + final DateTime lastDependencyChanged; + + @override + final DateTime finished; + + static const _$table = ( + tableName: 'tasks', + columns: [ + 'runtimeVersion', + 'package', + 'state', + 'pendingAt', + 'lastDependencyChanged', + 'finished', + ], + columnInfo: + < + ({ + ColumnType type, + bool isNotNull, + Object? defaultValue, + bool autoIncrement, + }) + >[ + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.dateTime, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.dateTime, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.dateTime, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ], + primaryKey: ['runtimeVersion', 'package'], + unique: >[], + foreignKeys: + < + ({ + String name, + List columns, + String referencedTable, + List referencedColumns, + }) + >[], + readRow: _$Task._$fromDatabase, + ); + + static Task? _$fromDatabase(RowReader row) { + final runtimeVersion = row.readString(); + final package = row.readString(); + final state = ExposedForCodeGen.customDataTypeOrNull( + row.readString(), + TaskState.fromDatabase, + ); + final pendingAt = row.readDateTime(); + final lastDependencyChanged = row.readDateTime(); + final finished = row.readDateTime(); + if (runtimeVersion == null && + package == null && + state == null && + pendingAt == null && + lastDependencyChanged == null && + finished == null) { + return null; + } + return _$Task._( + runtimeVersion!, + package!, + state!, + pendingAt!, + lastDependencyChanged!, + finished!, + ); + } + + @override + String toString() => + 'Task(runtimeVersion: "$runtimeVersion", package: "$package", state: "$state", pendingAt: "$pendingAt", lastDependencyChanged: "$lastDependencyChanged", finished: "$finished")'; +} + +/// Extension methods for table defined in [Task]. +extension TableTaskExt on Table { + /// Insert row into the `tasks` table. + /// + /// Returns a [InsertSingle] statement on which `.execute` must be + /// called for the row to be inserted. + InsertSingle insert({ + required Expr runtimeVersion, + required Expr package, + required Expr state, + required Expr pendingAt, + required Expr lastDependencyChanged, + required Expr finished, + }) => ExposedForCodeGen.insertInto( + table: this, + values: [ + runtimeVersion, + package, + state, + pendingAt, + lastDependencyChanged, + finished, + ], + ); + + /// Delete a single row from the `tasks` table, specified by + /// _primary key_. + /// + /// Returns a [DeleteSingle] statement on which `.execute()` must be + /// called for the row to be deleted. + /// + /// To delete multiple rows, using `.where()` to filter which rows + /// should be deleted. If you wish to delete all rows, use + /// `.where((_) => toExpr(true)).delete()`. + DeleteSingle delete(String runtimeVersion, String package) => + ExposedForCodeGen.deleteSingle( + byKey(runtimeVersion, package), + _$Task._$table, + ); +} + +/// Extension methods for building queries against the `tasks` table. +extension QueryTaskExt on Query<(Expr,)> { + /// Lookup a single row in `tasks` table using the _primary key_. + /// + /// Returns a [QuerySingle] object, which returns at-most one row, + /// when `.fetch()` is called. + QuerySingle<(Expr,)> byKey(String runtimeVersion, String package) => + where( + (task) => + task.runtimeVersion.equalsValue(runtimeVersion) & + task.package.equalsValue(package), + ).first; + + /// Update all rows in the `tasks` table matching this [Query]. + /// + /// The changes to be applied to each row matching this [Query] are + /// defined using the [updateBuilder], which is given an [Expr] + /// representation of the row being updated and a `set` function to + /// specify which fields should be updated. The result of the `set` + /// function should always be returned from the `updateBuilder`. + /// + /// Returns an [Update] statement on which `.execute()` must be called + /// for the rows to be updated. + /// + /// **Example:** decrementing `1` from the `value` field for each row + /// where `value > 0`. + /// ```dart + /// await db.mytable + /// .where((row) => row.value > toExpr(0)) + /// .update((row, set) => set( + /// value: row.value - toExpr(1), + /// )) + /// .execute(); + /// ``` + /// + /// > [!WARNING] + /// > The `updateBuilder` callback does not make the update, it builds + /// > the expressions for updating the rows. You should **never** invoke + /// > the `set` function more than once, and the result should always + /// > be returned immediately. + Update update( + UpdateSet Function( + Expr task, + UpdateSet Function({ + Expr runtimeVersion, + Expr package, + Expr state, + Expr pendingAt, + Expr lastDependencyChanged, + Expr finished, + }) + set, + ) + updateBuilder, + ) => ExposedForCodeGen.update( + this, + _$Task._$table, + (task) => updateBuilder( + task, + ({ + Expr? runtimeVersion, + Expr? package, + Expr? state, + Expr? pendingAt, + Expr? lastDependencyChanged, + Expr? finished, + }) => ExposedForCodeGen.buildUpdate([ + runtimeVersion, + package, + state, + pendingAt, + lastDependencyChanged, + finished, + ]), + ), + ); + + /// Delete all rows in the `tasks` table matching this [Query]. + /// + /// Returns a [Delete] statement on which `.execute()` must be called + /// for the rows to be deleted. + Delete delete() => ExposedForCodeGen.delete(this, _$Task._$table); +} + +/// Extension methods for building point queries against the `tasks` table. +extension QuerySingleTaskExt on QuerySingle<(Expr,)> { + /// Update the row (if any) in the `tasks` table matching this + /// [QuerySingle]. + /// + /// The changes to be applied to the row matching this [QuerySingle] are + /// defined using the [updateBuilder], which is given an [Expr] + /// representation of the row being updated and a `set` function to + /// specify which fields should be updated. The result of the `set` + /// function should always be returned from the `updateBuilder`. + /// + /// Returns an [UpdateSingle] statement on which `.execute()` must be + /// called for the row to be updated. The resulting statement will + /// **not** fail, if there are no rows matching this query exists. + /// + /// **Example:** decrementing `1` from the `value` field the row with + /// `id = 1`. + /// ```dart + /// await db.mytable + /// .byKey(1) + /// .update((row, set) => set( + /// value: row.value - toExpr(1), + /// )) + /// .execute(); + /// ``` + /// + /// > [!WARNING] + /// > The `updateBuilder` callback does not make the update, it builds + /// > the expressions for updating the rows. You should **never** invoke + /// > the `set` function more than once, and the result should always + /// > be returned immediately. + UpdateSingle update( + UpdateSet Function( + Expr task, + UpdateSet Function({ + Expr runtimeVersion, + Expr package, + Expr state, + Expr pendingAt, + Expr lastDependencyChanged, + Expr finished, + }) + set, + ) + updateBuilder, + ) => ExposedForCodeGen.updateSingle( + this, + _$Task._$table, + (task) => updateBuilder( + task, + ({ + Expr? runtimeVersion, + Expr? package, + Expr? state, + Expr? pendingAt, + Expr? lastDependencyChanged, + Expr? finished, + }) => ExposedForCodeGen.buildUpdate([ + runtimeVersion, + package, + state, + pendingAt, + lastDependencyChanged, + finished, + ]), + ), + ); + + /// Delete the row (if any) in the `tasks` table matching this [QuerySingle]. + /// + /// Returns a [DeleteSingle] statement on which `.execute()` must be called + /// for the row to be deleted. The resulting statement will **not** + /// fail, if there are no rows matching this query exists. + DeleteSingle delete() => + ExposedForCodeGen.deleteSingle(this, _$Task._$table); +} + +/// Extension methods for expressions on a row in the `tasks` table. +extension ExpressionTaskExt on Expr { + /// Runtime version this [Task] belongs to. + Expr get runtimeVersion => + ExposedForCodeGen.field(this, 0, ExposedForCodeGen.text); + + Expr get package => + ExposedForCodeGen.field(this, 1, ExposedForCodeGen.text); + + Expr get state => + ExposedForCodeGen.field(this, 2, TaskStateExt._exprType); + + /// Next [DateTime] at which point some package version becomes pending. + Expr get pendingAt => + ExposedForCodeGen.field(this, 3, ExposedForCodeGen.dateTime); + + /// Last [DateTime] a dependency was updated. + Expr get lastDependencyChanged => + ExposedForCodeGen.field(this, 4, ExposedForCodeGen.dateTime); + + /// The last time the a worker completed with a failure or success. + Expr get finished => + ExposedForCodeGen.field(this, 5, ExposedForCodeGen.dateTime); + + /// Get [SubQuery] of rows from the `taskDependencies` table which + /// reference this row. + /// + /// This returns a [SubQuery] of [TaskDependency] rows, + /// where [TaskDependency.runtimeVersion], [TaskDependency.package] + /// references [Task.runtimeVersion], [Task.package] + /// in this row. + SubQuery<(Expr,)> get dependencies => + ExposedForCodeGen.subqueryTable(_$TaskDependency._$table).where( + (r) => + r.runtimeVersion.equals(runtimeVersion) & r.package.equals(package), + ); +} + +extension ExpressionNullableTaskExt on Expr { + /// Runtime version this [Task] belongs to. + Expr get runtimeVersion => + ExposedForCodeGen.field(this, 0, ExposedForCodeGen.text); + + Expr get package => + ExposedForCodeGen.field(this, 1, ExposedForCodeGen.text); + + Expr get state => + ExposedForCodeGen.field(this, 2, TaskStateExt._exprType); + + /// Next [DateTime] at which point some package version becomes pending. + Expr get pendingAt => + ExposedForCodeGen.field(this, 3, ExposedForCodeGen.dateTime); + + /// Last [DateTime] a dependency was updated. + Expr get lastDependencyChanged => + ExposedForCodeGen.field(this, 4, ExposedForCodeGen.dateTime); + + /// The last time the a worker completed with a failure or success. + Expr get finished => + ExposedForCodeGen.field(this, 5, ExposedForCodeGen.dateTime); + + /// Get [SubQuery] of rows from the `taskDependencies` table which + /// reference this row. + /// + /// This returns a [SubQuery] of [TaskDependency] rows, + /// where [TaskDependency.runtimeVersion], [TaskDependency.package] + /// references [Task.runtimeVersion], [Task.package] + /// in this row, if any. + /// + /// If this row is `NULL` the subquery is always be empty. + SubQuery<(Expr,)> get dependencies => + ExposedForCodeGen.subqueryTable(_$TaskDependency._$table).where( + (r) => + r.runtimeVersion.equalsUnlessNull(runtimeVersion).asNotNull() & + r.package.equalsUnlessNull(package).asNotNull(), + ); + + /// Check if the row is not `NULL`. + /// + /// This will check if _primary key_ fields in this row are `NULL`. + /// + /// If this is a reference lookup by subquery it might be more efficient + /// to check if the referencing field is `NULL`. + Expr isNotNull() => runtimeVersion.isNotNull() & package.isNotNull(); + + /// Check if the row is `NULL`. + /// + /// This will check if _primary key_ fields in this row are `NULL`. + /// + /// If this is a reference lookup by subquery it might be more efficient + /// to check if the referencing field is `NULL`. + Expr isNull() => isNotNull().not(); +} + +extension InnerJoinTaskTaskDependencyExt + on InnerJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [Task.runtimeVersion] = [TaskDependency.runtimeVersion] and [Task.package] = [TaskDependency.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + a.runtimeVersion.equals(b.runtimeVersion) & a.package.equals(b.package), + ); +} + +extension LeftJoinTaskTaskDependencyExt + on LeftJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [Task.runtimeVersion] = [TaskDependency.runtimeVersion] and [Task.package] = [TaskDependency.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + a.runtimeVersion.equals(b.runtimeVersion) & a.package.equals(b.package), + ); +} + +extension RightJoinTaskTaskDependencyExt + on RightJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [Task.runtimeVersion] = [TaskDependency.runtimeVersion] and [Task.package] = [TaskDependency.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + a.runtimeVersion.equals(b.runtimeVersion) & a.package.equals(b.package), + ); +} + +final class _$TaskDependency extends TaskDependency { + _$TaskDependency._(this.runtimeVersion, this.package, this.dependency); + + @override + final String runtimeVersion; + + @override + final String package; + + @override + final String dependency; + + static const _$table = ( + tableName: 'taskDependencies', + columns: ['runtimeVersion', 'package', 'dependency'], + columnInfo: + < + ({ + ColumnType type, + bool isNotNull, + Object? defaultValue, + bool autoIncrement, + }) + >[ + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ( + type: ExposedForCodeGen.text, + isNotNull: true, + defaultValue: null, + autoIncrement: false, + ), + ], + primaryKey: ['runtimeVersion', 'package', 'dependency'], + unique: >[], + foreignKeys: + < + ({ + String name, + List columns, + String referencedTable, + List referencedColumns, + }) + >[ + ( + name: 'task', + columns: ['runtimeVersion', 'package'], + referencedTable: 'tasks', + referencedColumns: ['runtimeVersion', 'package'], + ), + ], + readRow: _$TaskDependency._$fromDatabase, + ); + + static TaskDependency? _$fromDatabase(RowReader row) { + final runtimeVersion = row.readString(); + final package = row.readString(); + final dependency = row.readString(); + if (runtimeVersion == null && package == null && dependency == null) { + return null; + } + return _$TaskDependency._(runtimeVersion!, package!, dependency!); + } + + @override + String toString() => + 'TaskDependency(runtimeVersion: "$runtimeVersion", package: "$package", dependency: "$dependency")'; +} + +/// Extension methods for table defined in [TaskDependency]. +extension TableTaskDependencyExt on Table { + /// Insert row into the `taskDependencies` table. + /// + /// Returns a [InsertSingle] statement on which `.execute` must be + /// called for the row to be inserted. + InsertSingle insert({ + required Expr runtimeVersion, + required Expr package, + required Expr dependency, + }) => ExposedForCodeGen.insertInto( + table: this, + values: [runtimeVersion, package, dependency], + ); + + /// Delete a single row from the `taskDependencies` table, specified by + /// _primary key_. + /// + /// Returns a [DeleteSingle] statement on which `.execute()` must be + /// called for the row to be deleted. + /// + /// To delete multiple rows, using `.where()` to filter which rows + /// should be deleted. If you wish to delete all rows, use + /// `.where((_) => toExpr(true)).delete()`. + DeleteSingle delete( + String runtimeVersion, + String package, + String dependency, + ) => ExposedForCodeGen.deleteSingle( + byKey(runtimeVersion, package, dependency), + _$TaskDependency._$table, + ); +} + +/// Extension methods for building queries against the `taskDependencies` table. +extension QueryTaskDependencyExt on Query<(Expr,)> { + /// Lookup a single row in `taskDependencies` table using the _primary key_. + /// + /// Returns a [QuerySingle] object, which returns at-most one row, + /// when `.fetch()` is called. + QuerySingle<(Expr,)> byKey( + String runtimeVersion, + String package, + String dependency, + ) => where( + (taskDependency) => + taskDependency.runtimeVersion.equalsValue(runtimeVersion) & + taskDependency.package.equalsValue(package) & + taskDependency.dependency.equalsValue(dependency), + ).first; + + /// Update all rows in the `taskDependencies` table matching this [Query]. + /// + /// The changes to be applied to each row matching this [Query] are + /// defined using the [updateBuilder], which is given an [Expr] + /// representation of the row being updated and a `set` function to + /// specify which fields should be updated. The result of the `set` + /// function should always be returned from the `updateBuilder`. + /// + /// Returns an [Update] statement on which `.execute()` must be called + /// for the rows to be updated. + /// + /// **Example:** decrementing `1` from the `value` field for each row + /// where `value > 0`. + /// ```dart + /// await db.mytable + /// .where((row) => row.value > toExpr(0)) + /// .update((row, set) => set( + /// value: row.value - toExpr(1), + /// )) + /// .execute(); + /// ``` + /// + /// > [!WARNING] + /// > The `updateBuilder` callback does not make the update, it builds + /// > the expressions for updating the rows. You should **never** invoke + /// > the `set` function more than once, and the result should always + /// > be returned immediately. + Update update( + UpdateSet Function( + Expr taskDependency, + UpdateSet Function({ + Expr runtimeVersion, + Expr package, + Expr dependency, + }) + set, + ) + updateBuilder, + ) => ExposedForCodeGen.update( + this, + _$TaskDependency._$table, + (taskDependency) => updateBuilder( + taskDependency, + ({ + Expr? runtimeVersion, + Expr? package, + Expr? dependency, + }) => ExposedForCodeGen.buildUpdate([ + runtimeVersion, + package, + dependency, + ]), + ), + ); + + /// Delete all rows in the `taskDependencies` table matching this [Query]. + /// + /// Returns a [Delete] statement on which `.execute()` must be called + /// for the rows to be deleted. + Delete delete() => + ExposedForCodeGen.delete(this, _$TaskDependency._$table); +} + +/// Extension methods for building point queries against the `taskDependencies` table. +extension QuerySingleTaskDependencyExt on QuerySingle<(Expr,)> { + /// Update the row (if any) in the `taskDependencies` table matching this + /// [QuerySingle]. + /// + /// The changes to be applied to the row matching this [QuerySingle] are + /// defined using the [updateBuilder], which is given an [Expr] + /// representation of the row being updated and a `set` function to + /// specify which fields should be updated. The result of the `set` + /// function should always be returned from the `updateBuilder`. + /// + /// Returns an [UpdateSingle] statement on which `.execute()` must be + /// called for the row to be updated. The resulting statement will + /// **not** fail, if there are no rows matching this query exists. + /// + /// **Example:** decrementing `1` from the `value` field the row with + /// `id = 1`. + /// ```dart + /// await db.mytable + /// .byKey(1) + /// .update((row, set) => set( + /// value: row.value - toExpr(1), + /// )) + /// .execute(); + /// ``` + /// + /// > [!WARNING] + /// > The `updateBuilder` callback does not make the update, it builds + /// > the expressions for updating the rows. You should **never** invoke + /// > the `set` function more than once, and the result should always + /// > be returned immediately. + UpdateSingle update( + UpdateSet Function( + Expr taskDependency, + UpdateSet Function({ + Expr runtimeVersion, + Expr package, + Expr dependency, + }) + set, + ) + updateBuilder, + ) => ExposedForCodeGen.updateSingle( + this, + _$TaskDependency._$table, + (taskDependency) => updateBuilder( + taskDependency, + ({ + Expr? runtimeVersion, + Expr? package, + Expr? dependency, + }) => ExposedForCodeGen.buildUpdate([ + runtimeVersion, + package, + dependency, + ]), + ), + ); + + /// Delete the row (if any) in the `taskDependencies` table matching this [QuerySingle]. + /// + /// Returns a [DeleteSingle] statement on which `.execute()` must be called + /// for the row to be deleted. The resulting statement will **not** + /// fail, if there are no rows matching this query exists. + DeleteSingle delete() => + ExposedForCodeGen.deleteSingle(this, _$TaskDependency._$table); +} + +/// Extension methods for expressions on a row in the `taskDependencies` table. +extension ExpressionTaskDependencyExt on Expr { + Expr get runtimeVersion => + ExposedForCodeGen.field(this, 0, ExposedForCodeGen.text); + + Expr get package => + ExposedForCodeGen.field(this, 1, ExposedForCodeGen.text); + + /// Name of a package that is either a direct or transitive dependency of + /// [package]. + Expr get dependency => + ExposedForCodeGen.field(this, 2, ExposedForCodeGen.text); + + /// Do a subquery lookup of the row from table + /// `tasks` referenced in + /// [runtimeVersion], [package]. + /// + /// The gets the row from table `tasks` where + /// [Task.runtimeVersion], [Task.package] + /// is equal to [runtimeVersion], [package]. + Expr get task => ExposedForCodeGen.subqueryTable(_$Task._$table) + .where( + (r) => + r.runtimeVersion.equals(runtimeVersion) & r.package.equals(package), + ) + .first + .asNotNull(); +} + +extension ExpressionNullableTaskDependencyExt on Expr { + Expr get runtimeVersion => + ExposedForCodeGen.field(this, 0, ExposedForCodeGen.text); + + Expr get package => + ExposedForCodeGen.field(this, 1, ExposedForCodeGen.text); + + /// Name of a package that is either a direct or transitive dependency of + /// [package]. + Expr get dependency => + ExposedForCodeGen.field(this, 2, ExposedForCodeGen.text); + + /// Do a subquery lookup of the row from table + /// `tasks` referenced in + /// [runtimeVersion], [package]. + /// + /// The gets the row from table `tasks` where + /// [Task.runtimeVersion], [Task.package] + /// is equal to [runtimeVersion], [package], if any. + /// + /// If this row is `NULL` the subquery is always return `NULL`. + Expr get task => ExposedForCodeGen.subqueryTable(_$Task._$table) + .where( + (r) => + r.runtimeVersion.equalsUnlessNull(runtimeVersion).asNotNull() & + r.package.equalsUnlessNull(package).asNotNull(), + ) + .first; + + /// Check if the row is not `NULL`. + /// + /// This will check if _primary key_ fields in this row are `NULL`. + /// + /// If this is a reference lookup by subquery it might be more efficient + /// to check if the referencing field is `NULL`. + Expr isNotNull() => + runtimeVersion.isNotNull() & package.isNotNull() & dependency.isNotNull(); + + /// Check if the row is `NULL`. + /// + /// This will check if _primary key_ fields in this row are `NULL`. + /// + /// If this is a reference lookup by subquery it might be more efficient + /// to check if the referencing field is `NULL`. + Expr isNull() => isNotNull().not(); +} + +extension InnerJoinTaskDependencyTaskExt + on InnerJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [TaskDependency.runtimeVersion] = [Task.runtimeVersion] and [TaskDependency.package] = [Task.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + b.runtimeVersion.equals(a.runtimeVersion) & b.package.equals(a.package), + ); +} + +extension LeftJoinTaskDependencyTaskExt + on LeftJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [TaskDependency.runtimeVersion] = [Task.runtimeVersion] and [TaskDependency.package] = [Task.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + b.runtimeVersion.equals(a.runtimeVersion) & b.package.equals(a.package), + ); +} + +extension RightJoinTaskDependencyTaskExt + on RightJoin<(Expr,), (Expr,)> { + /// Join using the `task` _foreign key_. + /// + /// This will match rows where [TaskDependency.runtimeVersion] = [Task.runtimeVersion] and [TaskDependency.package] = [Task.package]. + Query<(Expr, Expr)> usingTask() => on( + (a, b) => + b.runtimeVersion.equals(a.runtimeVersion) & b.package.equals(a.package), + ); +} + +/// Wrap this [TaskState] as [Expr] for use queries with +/// `package:typed_sql`. +extension TaskStateExt on TaskState { + static final _exprType = ExposedForCodeGen.customDataType( + ExposedForCodeGen.text, + TaskState.fromDatabase, + ); + + /// Wrap this [TaskState] as [Expr] for use queries with + /// `package:typed_sql`. + Expr get asExpr => + ExposedForCodeGen.literalCustomDataType(this, _exprType).asNotNull(); +} + +/// Wrap this [TaskState] as [Expr] for use queries with +/// `package:typed_sql`. +extension TaskStateNullableExt on TaskState? { + /// Wrap this [TaskState] as [Expr] for use queries with + /// `package:typed_sql`. + Expr get asExpr => + ExposedForCodeGen.literalCustomDataType(this, TaskStateExt._exprType); +} diff --git a/app/lib/database/schema.task.dart b/app/lib/database/schema.task.dart new file mode 100644 index 0000000000..c2eb237524 --- /dev/null +++ b/app/lib/database/schema.task.dart @@ -0,0 +1,62 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +part of 'schema.dart'; + +@PrimaryKey(['runtimeVersion', 'package']) +abstract final class Task extends Row { + /// Runtime version this [Task] belongs to. + String get runtimeVersion; + String get package; + + TaskState get state; + + /// Next [DateTime] at which point some package version becomes pending. + DateTime get pendingAt; + + /// Last [DateTime] a dependency was updated. + DateTime get lastDependencyChanged; + + /// The last time the a worker completed with a failure or success. + DateTime get finished; +} + +@PrimaryKey(['runtimeVersion', 'package', 'dependency']) +@ForeignKey( + ['runtimeVersion', 'package'], + table: 'tasks', + fields: ['runtimeVersion', 'package'], + name: 'task', + as: 'dependencies', +) +abstract final class TaskDependency extends Row { + String get runtimeVersion; + String get package; + + /// Name of a package that is either a direct or transitive dependency of + /// [package]. + String get dependency; +} + +@immutable +@JsonSerializable() +final class TaskState implements CustomDataType { + /// Scheduling state for all versions of this package. + final Map versions; + + /// The list of tokens that were removed from this [versions]. + /// When a worker reports back using one of these tokens, they will + /// recieve a [TaskAbortedException]. + final List abortedTokens; + + TaskState({required this.versions, required this.abortedTokens}); + + factory TaskState.fromJson(Map m) => _$TaskStateFromJson(m); + Map toJson() => _$TaskStateToJson(this); + + factory TaskState.fromDatabase(String value) => + TaskState.fromJson(json.decode(value) as Map); + @override + String toDatabase() => json.encode(toJson()); +} diff --git a/app/lib/frontend/handlers/pubapi.client.dart b/app/lib/frontend/handlers/pubapi.client.dart index fde241e21d..951c4edd9b 100644 --- a/app/lib/frontend/handlers/pubapi.client.dart +++ b/app/lib/frontend/handlers/pubapi.client.dart @@ -1,5 +1,5 @@ -// dart format width=80 // GENERATED CODE - DO NOT MODIFY BY HAND +// dart format width=80 // ************************************************************************** // ClientLibraryGenerator diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 2db702cf2b..f977613a9d 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -43,6 +43,7 @@ dependencies: shelf_router: ^1.1.0 stack_trace: ^1.10.0 stream_transform: ^2.0.0 + typed_sql: ^0.1.4 watcher: ^1.0.0 yaml: ^3.1.0 # pana version to be pinned @@ -59,7 +60,10 @@ dev_dependencies: build_verify: ^3.0.0 coverage: any # test already depends on it json_serializable: ^6.0.1 - shelf_router_generator: ^1.0.0 - source_gen: '^3.0.0' + shelf_router_generator: ^1.1.3 + source_gen: '^4.0.0' test: ^1.16.5 xml: ^6.0.0 + +dependency_overrides: + build: ^4.0.0 diff --git a/app/test/database/postgresql_ci_test.dart b/app/test/database/postgresql_ci_test.dart index e8dcc4ddd4..320f61232a 100644 --- a/app/test/database/postgresql_ci_test.dart +++ b/app/test/database/postgresql_ci_test.dart @@ -5,8 +5,12 @@ import 'package:clock/clock.dart'; import 'package:postgres/postgres.dart'; import 'package:pub_dev/database/database.dart'; +import 'package:pub_dev/database/schema.dart'; import 'package:pub_dev/shared/env_config.dart'; +import 'package:pub_dev/shared/versions.dart'; +import 'package:pub_dev/task/models.dart'; import 'package:test/test.dart'; +import 'package:typed_sql/typed_sql.dart'; import '../shared/test_services.dart'; @@ -43,5 +47,41 @@ void main() { expect(name, contains('fake_pub_')); }, ); + + testWithProfile( + 'typed schema access', + fn: () async { + final db = primaryDatabase!.db; + await db.tasks + .insert( + runtimeVersion: runtimeVersion.asExpr, + package: 'foo'.asExpr, + state: TaskState( + versions: { + '1.0.0': PackageVersionStateInfo( + scheduled: clock.now(), + attempts: 0, + ), + }, + abortedTokens: [], + ).asExpr, + pendingAt: clock.now().asExpr, + lastDependencyChanged: clock.now().asExpr, + finished: clock.now().asExpr, + ) + .execute(); + + final rows = await db.tasks + .select((b) => (b.package, b.runtimeVersion, b.state)) + .fetch(); + expect(rows, hasLength(1)); + expect(rows.first.$1, 'foo'); + expect(rows.first.$2, runtimeVersion); + expect(rows.first.$3.toJson(), { + 'versions': {'1.0.0': isNotNull}, + 'abortedTokens': [], + }); + }, + ); }); } diff --git a/pkg/_pub_shared/lib/src/pubapi.client.dart b/pkg/_pub_shared/lib/src/pubapi.client.dart index fde241e21d..951c4edd9b 100644 --- a/pkg/_pub_shared/lib/src/pubapi.client.dart +++ b/pkg/_pub_shared/lib/src/pubapi.client.dart @@ -1,5 +1,5 @@ -// dart format width=80 // GENERATED CODE - DO NOT MODIFY BY HAND +// dart format width=80 // ************************************************************************** // ClientLibraryGenerator diff --git a/pkg/_pub_shared/pubspec.yaml b/pkg/_pub_shared/pubspec.yaml index 3add4ced09..00b1d53708 100644 --- a/pkg/_pub_shared/pubspec.yaml +++ b/pkg/_pub_shared/pubspec.yaml @@ -22,5 +22,5 @@ dev_dependencies: build_verify: ^3.0.0 coverage: any # test already depends on it json_serializable: ^6.0.1 - source_gen: '^3.0.0' + source_gen: '^4.0.0' test: '^1.16.5' diff --git a/pkg/api_builder/lib/src/api_router_generator.dart b/pkg/api_builder/lib/src/api_router_generator.dart index 9e0f60b8a8..d62ffbf1c8 100644 --- a/pkg/api_builder/lib/src/api_router_generator.dart +++ b/pkg/api_builder/lib/src/api_router_generator.dart @@ -14,7 +14,7 @@ import 'package:source_gen/source_gen.dart' as g; import 'shared.dart' show EndPointGenerator, Handler; // Type checkers that we need later -final _responseType = g.TypeChecker.fromRuntime(shelf.Response); +final _responseType = g.TypeChecker.typeNamed(shelf.Response); class ApiRouterGenerator extends EndPointGenerator { @override diff --git a/pkg/api_builder/lib/src/client_library_generator.dart b/pkg/api_builder/lib/src/client_library_generator.dart index aff6f75142..218d213bb9 100644 --- a/pkg/api_builder/lib/src/client_library_generator.dart +++ b/pkg/api_builder/lib/src/client_library_generator.dart @@ -20,7 +20,7 @@ code.Reference _referToType(DartType type) => code.refer( type.element!.firstFragment.libraryFragment!.source.uri.toString(), ); -final _responseType = g.TypeChecker.fromRuntime(shelf.Response); +final _responseType = g.TypeChecker.typeNamed(shelf.Response); /// Use the first Handler when a method has multiple EndPoint annotations. Iterable _removeDuplicateHandlers(Iterable handlers) { diff --git a/pkg/api_builder/lib/src/shared.dart b/pkg/api_builder/lib/src/shared.dart index 5a567d1dae..f6d1b00b8b 100644 --- a/pkg/api_builder/lib/src/shared.dart +++ b/pkg/api_builder/lib/src/shared.dart @@ -15,7 +15,7 @@ import 'package:source_gen/source_gen.dart' as g; import '../api_builder.dart' show EndPoint; // Type checkers that we need later -final _endPointType = g.TypeChecker.fromRuntime(EndPoint); +final _endPointType = g.TypeChecker.typeNamed(EndPoint); /// A representation of a handler that was annotated with [EndPoint]. class Handler { diff --git a/pkg/api_builder/pubspec.yaml b/pkg/api_builder/pubspec.yaml index 36c4632030..1e98a54f00 100644 --- a/pkg/api_builder/pubspec.yaml +++ b/pkg/api_builder/pubspec.yaml @@ -8,10 +8,10 @@ resolution: workspace dependencies: analyzer: ^8.0.0 - build: ^3.0.0 + build: ^4.0.0 build_config: ^1.0.0 collection: ^1.18.0 - source_gen: ^3.0.0 + source_gen: ^4.0.0 shelf_router: ^1.0.0 code_builder: ^4.0.0 shelf: ^1.0.0 diff --git a/pubspec.lock b/pubspec.lock index 784915f589..428274a454 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -13,18 +13,18 @@ packages: dependency: transitive description: name: _fe_analyzer_shared - sha256: f0bb5d1648339c8308cc0b9838d8456b3cfe5c91f9dc1a735b4d003269e5da9a + sha256: c209688d9f5a5f26b2fb47a188131a6fb9e876ae9e47af3737c0b4f58a93470d url: "https://pub.dev" source: hosted - version: "88.0.0" + version: "91.0.0" analyzer: dependency: transitive description: name: analyzer - sha256: "0b7b9c329d2879f8f05d6c05b32ee9ec025f39b077864bdb5ac9a7b63418a98f" + sha256: f51c8499b35f9b26820cfe914828a6a98a94efd5cc78b37bb7d03debae3a1d08 url: "https://pub.dev" source: hosted - version: "8.1.1" + version: "8.4.1" appengine: dependency: transitive description: @@ -82,21 +82,21 @@ packages: source: hosted version: "1.2.3" build: - dependency: transitive + dependency: "direct overridden" description: name: build - sha256: "7174c5d84b0fed00a1f5e7543597b35d67560465ae3d909f0889b8b20419d5e3" + sha256: dfb67ccc9a78c642193e0c2d94cb9e48c2c818b3178a86097d644acdcde6a8d9 url: "https://pub.dev" source: hosted - version: "3.0.1" + version: "4.0.2" build_config: dependency: transitive description: name: build_config - sha256: "4ae2de3e1e67ea270081eaee972e1bd8f027d459f249e0f1186730784c2e7e33" + sha256: "4f64382b97504dc2fcdf487d5aae33418e08b4703fc21249e4db6d804a4d0187" url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.2.0" build_daemon: dependency: transitive description: @@ -105,30 +105,14 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.4" - build_resolvers: - dependency: transitive - description: - name: build_resolvers - sha256: "82730bf3d9043366ba8c02e4add05842a10739899520a6a22ddbd22d333bd5bb" - url: "https://pub.dev" - source: hosted - version: "3.0.1" build_runner: dependency: transitive description: name: build_runner - sha256: "32c6b3d172f1f46b7c4df6bc4a47b8d88afb9e505dd4ace4af80b3c37e89832b" + sha256: a9461b8e586bf018dd4afd2e13b49b08c6a844a4b226c8d1d10f3a723cdd78c3 url: "https://pub.dev" source: hosted - version: "2.6.1" - build_runner_core: - dependency: transitive - description: - name: build_runner_core - sha256: "4b188774b369104ad96c0e4ca2471e5162f0566ce277771b179bed5eabf2d048" - url: "https://pub.dev" - source: hosted - version: "9.2.1" + version: "2.10.1" build_verify: dependency: transitive description: @@ -757,10 +741,10 @@ packages: dependency: transitive description: name: shelf_router_generator - sha256: "03e5c598a7bad451f7f6cbd2867114023f7a677784f9f5ce38826091ab33de07" + sha256: "310416e0eb5a96c8b27f2586367f07b09dc480af06485c1f7951bbed1e8b8b08" url: "https://pub.dev" source: hosted - version: "1.1.2" + version: "1.1.3" shelf_static: dependency: transitive description: @@ -789,18 +773,18 @@ packages: dependency: transitive description: name: source_gen - sha256: "7b19d6ba131c6eb98bfcbf8d56c1a7002eba438af2e7ae6f8398b2b0f4f381e3" + sha256: "9098ab86015c4f1d8af6486b547b11100e73b193e1899015033cb3e14ad20243" url: "https://pub.dev" source: hosted - version: "3.1.0" + version: "4.0.2" source_helper: dependency: transitive description: name: source_helper - sha256: a447acb083d3a5ef17f983dd36201aeea33fedadb3228fa831f2f0c92f0f3aca + sha256: "6a3c6cc82073a8797f8c4dc4572146114a39652851c157db37e964d9c7038723" url: "https://pub.dev" source: hosted - version: "1.3.7" + version: "1.3.8" source_map_stack_trace: dependency: transitive description: @@ -833,6 +817,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: f18fd9a72d7a1ad2920db61368f2a69368f1cc9b56b8233e9d83b47b0a8435aa + url: "https://pub.dev" + source: hosted + version: "2.9.3" stack_trace: dependency: transitive description: @@ -913,14 +905,6 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.1" - timing: - dependency: transitive - description: - name: timing - sha256: "62ee18aca144e4a9f29d212f5a4c6a053be252b895ab14b5821996cff4ed90fe" - url: "https://pub.dev" - source: hosted - version: "1.0.2" typed_data: dependency: transitive description: @@ -929,6 +913,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.0" + typed_sql: + dependency: transitive + description: + name: typed_sql + sha256: fd05a4672e73f2f49bdfcaa5393ca68e0ce9ca4e4c15f60cf3585e157953d112 + url: "https://pub.dev" + source: hosted + version: "0.1.4" ulid: dependency: transitive description: