diff --git a/examples/src/EF/EF.csproj b/examples/src/EF/EF.csproj index 6c9eb3a4..6be5b878 100644 --- a/examples/src/EF/EF.csproj +++ b/examples/src/EF/EF.csproj @@ -11,5 +11,4 @@ - diff --git a/examples/src/EF/Program.cs b/examples/src/EF/Program.cs index 39a3e851..4b92a857 100644 --- a/examples/src/EF/Program.cs +++ b/examples/src/EF/Program.cs @@ -3,6 +3,7 @@ await using var db = new BloggingContext(); +await db.Database.EnsureDeletedAsync(); await db.Database.EnsureCreatedAsync(); Console.WriteLine("Inserting a new blog"); diff --git a/examples/src/EF_YC/Program.cs b/examples/src/EF_YC/Program.cs index 1a044137..a38068ca 100644 --- a/examples/src/EF_YC/Program.cs +++ b/examples/src/EF_YC/Program.cs @@ -19,7 +19,8 @@ await Parser.Default.ParseArguments(args).WithParsedAsync(async cmd .Options; await using var db = new AppDbContext(options); - db.Database.EnsureCreated(); + await db.Database.EnsureDeletedAsync(); + await db.Database.EnsureCreatedAsync(); db.Users.Add(new User { Name = "Alex", Email = "alex@example.com" }); db.Users.Add(new User { Name = "Kirill", Email = "kirill@example.com" }); diff --git a/src/EfCore.Ydb/src/Migrations/Internal/YdbHistoryRepository.cs b/src/EfCore.Ydb/src/Migrations/Internal/YdbHistoryRepository.cs index 61f66425..ccf536a4 100644 --- a/src/EfCore.Ydb/src/Migrations/Internal/YdbHistoryRepository.cs +++ b/src/EfCore.Ydb/src/Migrations/Internal/YdbHistoryRepository.cs @@ -1,19 +1,27 @@ using System; -using System.Data; +using System.Collections.Generic; using System.Diagnostics; -using System.Linq; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using EfCore.Ydb.Storage.Internal; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Microsoft.Extensions.Logging; +using Ydb.Sdk; using Ydb.Sdk.Ado; namespace EfCore.Ydb.Migrations.Internal; -// ReSharper disable once ClassNeverInstantiated.Global -public class YdbHistoryRepository(HistoryRepositoryDependencies dependencies) : HistoryRepository(dependencies) +public class YdbHistoryRepository(HistoryRepositoryDependencies dependencies) + : HistoryRepository(dependencies), IHistoryRepository { + private const string LockKey = "LockMigration"; + private const int ReleaseMaxAttempt = 10; + + private static readonly TimeSpan LockTimeout = TimeSpan.FromMinutes(2); + protected override bool InterpretExistsResult(object? value) => throw new InvalidOperationException("Shouldn't be called"); @@ -25,152 +33,168 @@ public override async Task AcquireDatabaseLockAsync( ) { Dependencies.MigrationsLogger.AcquiringMigrationLock(); - var dbLock = - new YdbMigrationDatabaseLock("migrationLock", this, (YdbRelationalConnection)Dependencies.Connection); - await dbLock.Lock(timeoutInSeconds: 60, cancellationToken); - return dbLock; - } - public override string GetCreateIfNotExistsScript() - => GetCreateScript().Replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS"); + var deadline = DateTime.UtcNow + LockTimeout; + DateTime now; - public override LockReleaseBehavior LockReleaseBehavior => LockReleaseBehavior.Transaction; - - protected override string ExistsSql - => throw new UnreachableException("Shouldn't be called. We check if exists using different approach"); + do + { + now = DateTime.UtcNow; - public override bool Exists() - => ExistsAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + try + { + await Dependencies.MigrationCommandExecutor.ExecuteNonQueryAsync( + AcquireDatabaseLockCommand(), + ((IYdbRelationalConnection)Dependencies.Connection).Clone(), // TODO usage ExecutionContext + new MigrationExecutionState(), + commitTransaction: true, + cancellationToken: cancellationToken + ).ConfigureAwait(false); + + return new YdbMigrationDatabaseLock(this); + } + catch (YdbException) + { + await Task.Delay(100 + Random.Shared.Next(1000), cancellationToken); + } + } while (now < deadline); - public override Task ExistsAsync(CancellationToken cancellationToken = default) - { - var connection = (YdbRelationalConnection)Dependencies.Connection; - var schema = (YdbConnection)connection.DbConnection; - var tables = schema.GetSchema("tables"); - - var foundTables = - from table in tables.AsEnumerable() - where table.Field("table_type") == "TABLE" - && table.Field("table_name") == TableName - select table; - return Task.FromResult(foundTables.Count() == 1); + throw new YdbException("Unable to obtain table lock - another EF instance may be running"); } - public override string GetBeginIfNotExistsScript(string migrationId) => throw new NotImplementedException(); - - public override string GetBeginIfExistsScript(string migrationId) => throw new NotImplementedException(); - - public override string GetEndIfScript() => throw new NotImplementedException(); - - private sealed class YdbMigrationDatabaseLock( - string name, - IHistoryRepository historyRepository, - YdbRelationalConnection ydbConnection - ) : IMigrationsDatabaseLock - { - private IYdbRelationalConnection Connection { get; } = ydbConnection.Clone(); - private volatile string _pid = null!; - private CancellationTokenSource? _watchDogToken; - - public async Task Lock(int timeoutInSeconds, CancellationToken cancellationToken = default) + private IReadOnlyList AcquireDatabaseLockCommand() => + Dependencies.MigrationsSqlGenerator.Generate(new List { - if (_watchDogToken != null) - { - throw new InvalidOperationException("Already locked"); - } - - await Connection.OpenAsync(cancellationToken); - await using (var command = Connection.DbConnection.CreateCommand()) + new SqlOperation { - command.CommandText = """ - CREATE TABLE IF NOT EXISTS shedlock ( - name Text NOT NULL, - locked_at Timestamp NOT NULL, - lock_until Timestamp NOT NULL, - locked_by Text NOT NULL, - PRIMARY KEY(name) - ); - """; - await command.ExecuteNonQueryAsync(cancellationToken); + Sql = GetInsertScript( + new HistoryRow( + LockKey, + $"LockTime: {DateTime.UtcNow.ToString(CultureInfo.InvariantCulture)}, PID: {Environment.ProcessId}" + ) + ) } + }); - _pid = $"PID:{Environment.ProcessId}"; + private async Task ReleaseDatabaseLockAsync() + { + for (var i = 0; i < ReleaseMaxAttempt; i++) + { + await using var connection = ((IYdbRelationalConnection)Dependencies.Connection).Clone().DbConnection; - var lockAcquired = false; - for (var i = 0; i < 10; i++) + try { - if (await UpdateLock(name, timeoutInSeconds)) - { - lockAcquired = true; - break; - } + await Dependencies.MigrationCommandExecutor.ExecuteNonQueryAsync( + ReleaseDatabaseLockCommand(), + ((IYdbRelationalConnection)Dependencies.Connection).Clone() + ).ConfigureAwait(false); - await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken); + return; } - - if (!lockAcquired) + catch (YdbException e) { - throw new TimeoutException("Failed to acquire lock for migration`"); + Dependencies.MigrationsLogger.Logger.LogError(e, "Failed release database lock"); } + } + } - _watchDogToken = new CancellationTokenSource(); - _ = Task.Run((async Task () => - { - while (true) - { - // ReSharper disable once PossibleLossOfFraction - await Task.Delay(TimeSpan.FromSeconds(timeoutInSeconds / 2), _watchDogToken.Token); - await UpdateLock(name, timeoutInSeconds); - } - // ReSharper disable once FunctionNeverReturns - })!, _watchDogToken.Token); + private IReadOnlyList ReleaseDatabaseLockCommand() => + Dependencies.MigrationsSqlGenerator.Generate(new List + { new SqlOperation { Sql = GetDeleteScript(LockKey) } } + ); + + bool IHistoryRepository.CreateIfNotExists() => CreateIfNotExistsAsync().GetAwaiter().GetResult(); + + public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) + { + if (await ExistsAsync(cancellationToken)) + { + return false; } - private async Task UpdateLock(string nameLock, int timeoutInSeconds) + try { - var command = Connection.DbConnection.CreateCommand(); - command.CommandText = - $""" - UPSERT INTO shedlock (name, locked_at, lock_until, locked_by) - VALUES ( - @name, - CurrentUtcTimestamp(), - Unwrap(CurrentUtcTimestamp() + Interval("PT{timeoutInSeconds}S")), - @locked_by - ); - """; - command.Parameters.Add(new YdbParameter("name", DbType.String, nameLock)); - command.Parameters.Add(new YdbParameter("locked_by", DbType.String, _pid)); + await Dependencies.MigrationCommandExecutor.ExecuteNonQueryAsync( + GetCreateIfNotExistsCommands(), + Dependencies.Connection, + cancellationToken: cancellationToken + ).ConfigureAwait(false); - try + return true; + } + catch (YdbException e) + { + if (e.Code == StatusCode.Overloaded) { - await command.ExecuteNonQueryAsync(); return true; } - catch (YdbException) + + throw; + } + } + + private IReadOnlyList GetCreateIfNotExistsCommands() => + Dependencies.MigrationsSqlGenerator.Generate(new List + { + new SqlOperation { - return false; + Sql = GetCreateIfNotExistsScript(), + SuppressTransaction = true } - } + }); - public void Dispose() - => DisposeInternalAsync().GetAwaiter().GetResult(); + public override string GetCreateIfNotExistsScript() + => GetCreateScript().Replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS"); - public async ValueTask DisposeAsync() - => await DisposeInternalAsync(); + public override LockReleaseBehavior LockReleaseBehavior => LockReleaseBehavior.Transaction; - private async Task DisposeInternalAsync() + protected override string ExistsSql + => throw new UnreachableException("Shouldn't be called. We check if exists using different approach"); + + public override bool Exists() + => ExistsAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + + public override async Task ExistsAsync(CancellationToken cancellationToken = default) + { + try { - if (_watchDogToken != null) + await Dependencies.MigrationCommandExecutor.ExecuteNonQueryAsync( + SelectHistoryTableCommand(), + Dependencies.Connection, + new MigrationExecutionState(), + commitTransaction: true, + cancellationToken: cancellationToken + ).ConfigureAwait(false); + + return true; + } + catch (YdbException) + { + return false; + } + } + + private IReadOnlyList SelectHistoryTableCommand() => + Dependencies.MigrationsSqlGenerator.Generate(new List + { + new SqlOperation { - await _watchDogToken.CancelAsync(); + Sql = $"SELECT * FROM {SqlGenerationHelper.DelimitIdentifier(TableName, TableSchema)}" + + $" WHERE MigrationId = '{LockKey}';" } + }); - _watchDogToken = null; - await using var connection = Connection.DbConnection.CreateCommand(); - connection.CommandText = "DELETE FROM shedlock WHERE name = '{_name}' AND locked_by = '{PID}';"; - await connection.ExecuteNonQueryAsync(); - } + public override string GetBeginIfNotExistsScript(string migrationId) => throw new NotSupportedException(); + + public override string GetBeginIfExistsScript(string migrationId) => throw new NotSupportedException(); + + public override string GetEndIfScript() => throw new NotSupportedException(); + + private sealed class YdbMigrationDatabaseLock(YdbHistoryRepository historyRepository) : IMigrationsDatabaseLock + { + public void Dispose() => historyRepository.ReleaseDatabaseLockAsync().GetAwaiter().GetResult(); + + public async ValueTask DisposeAsync() => await historyRepository.ReleaseDatabaseLockAsync(); public IHistoryRepository HistoryRepository { get; } = historyRepository; } diff --git a/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs b/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs index 286c718f..10e89ab3 100644 --- a/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs +++ b/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs @@ -1,38 +1,86 @@ using System; +using System.Data; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.Migrations.Operations; using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging; using Ydb.Sdk.Ado; namespace EfCore.Ydb.Storage.Internal; -public class YdbDatabaseCreator( - RelationalDatabaseCreatorDependencies dependencies -) : RelationalDatabaseCreator(dependencies) +public class YdbDatabaseCreator(RelationalDatabaseCreatorDependencies dependencies) + : RelationalDatabaseCreator(dependencies) { - public override bool Exists() - => ExistsInternal().GetAwaiter().GetResult(); + public override bool Exists() => ExistsAsync().GetAwaiter().GetResult(); - public override Task ExistsAsync(CancellationToken cancellationToken = new()) - => ExistsInternal(cancellationToken); - - private async Task ExistsInternal(CancellationToken cancellationToken = default) + public override async Task ExistsAsync(CancellationToken cancellationToken = default) { await using var connection = Dependencies.Connection; + try { await connection.OpenAsync(cancellationToken, errorsExpected: true); return true; } + catch (YdbException e) + { + Dependencies.CommandLogger.Logger.LogCritical(e, "Failed to verify database existence"); + + return false; + } + } + + public override bool HasTables() => HasTablesAsync().GetAwaiter().GetResult(); + + public override async Task HasTablesAsync(CancellationToken cancellationToken = default) + { + await using var connection = Dependencies.Connection; + + try + { + await connection.OpenAsync(cancellationToken, errorsExpected: true); + + var dataTable = await connection + .DbConnection + .GetSchemaAsync("Tables", [null, "TABLE"], cancellationToken); + + return dataTable.Rows.Count > 0; + } catch (YdbException) { return false; } } - public override bool HasTables() => false; + public override void Create() => CreateAsync().GetAwaiter().GetResult(); + + public override async Task CreateAsync(CancellationToken cancellationToken = default) + { + if (await ExistsAsync(cancellationToken)) + { + return; + } - public override void Create() => throw new NotSupportedException("YDB does not support database creation"); + throw new NotSupportedException("YDB does not support database creation"); + } - public override void Delete() => throw new NotSupportedException("YDB does not support database deletion"); + public override void Delete() => DeleteAsync().GetAwaiter().GetResult(); + + public override async Task DeleteAsync(CancellationToken cancellationToken = default) + { + await using var connection = Dependencies.Connection; + await connection.OpenAsync(cancellationToken); + + var dataTable = await connection + .DbConnection + .GetSchemaAsync("Tables", [null, "TABLE"], cancellationToken); + + var dropTableOperations = (from DataRow row in dataTable.Rows + select new DropTableOperation { Name = row["table_name"].ToString() }).ToList(); + + await Dependencies.MigrationCommandExecutor.ExecuteNonQueryAsync(Dependencies.MigrationsSqlGenerator + .Generate(dropTableOperations), connection, cancellationToken); + } } diff --git a/src/EfCore.Ydb/src/Storage/Internal/YdbRelationalConnection.cs b/src/EfCore.Ydb/src/Storage/Internal/YdbRelationalConnection.cs index 1d6fa7c9..186645e9 100644 --- a/src/EfCore.Ydb/src/Storage/Internal/YdbRelationalConnection.cs +++ b/src/EfCore.Ydb/src/Storage/Internal/YdbRelationalConnection.cs @@ -36,8 +36,12 @@ protected override DbConnection CreateDbConnection() public IYdbRelationalConnection Clone() { - var connectionStringBuilder = new YdbConnectionStringBuilder(GetValidatedConnectionString()); - var options = new DbContextOptionsBuilder().UseYdb(connectionStringBuilder.ToString()).Options; + var options = new DbContextOptionsBuilder().UseYdb(GetValidatedConnectionString(), builder => + { + builder.WithCredentialsProvider(_credentialsProvider); + builder.WithServerCertificates(_serverCertificates); + }).Options; + return new YdbRelationalConnection(Dependencies with { ContextOptions = options }); } } diff --git a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsInfrastructureTest.cs b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsInfrastructureTest.cs new file mode 100644 index 00000000..0de0e79a --- /dev/null +++ b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsInfrastructureTest.cs @@ -0,0 +1,103 @@ +using EfCore.Ydb.FunctionalTests.TestUtilities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.TestUtilities; +using Xunit; + +namespace EfCore.Ydb.FunctionalTests.Migrations; + +public class YdbMigrationsInfrastructureTest(YdbMigrationsInfrastructureTest.YdbMigrationsInfrastructureFixture fixture) + : MigrationsInfrastructureTestBase(fixture) +{ + public class YdbMigrationsInfrastructureFixture : MigrationsInfrastructureFixtureBase + { + protected override ITestStoreFactory TestStoreFactory => YdbTestStoreFactory.Instance; + } + + protected override void GiveMeSomeTime(DbContext db) + { + } + + protected override Task GiveMeSomeTimeAsync(DbContext db) => Task.CompletedTask; + + [ConditionalFact(Skip = "TODO")] + public override void Can_diff_against_2_2_model() + { + } + + [ConditionalFact(Skip = "TODO")] + public override void Can_diff_against_3_0_ASP_NET_Identity_model() + { + } + + [ConditionalFact(Skip = "TODO")] + public override void Can_diff_against_2_2_ASP_NET_Identity_model() + { + } + + [ConditionalFact(Skip = "TODO")] + public override void Can_diff_against_2_1_ASP_NET_Identity_model() + { + } + + [ConditionalFact(Skip = "TODO")] + public override void Can_apply_all_migrations() => base.Can_apply_all_migrations(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_apply_all_migrations_async() => base.Can_apply_all_migrations_async(); + + [ConditionalFact(Skip = "TODO")] + public override void Can_apply_range_of_migrations() => base.Can_apply_range_of_migrations(); + + [ConditionalFact(Skip = "TODO")] + public override void Can_apply_second_migration_in_parallel() => base.Can_apply_second_migration_in_parallel(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_apply_second_migration_in_parallel_async() => + base.Can_apply_second_migration_in_parallel_async(); + + [ConditionalFact(Skip = "TODO")] + public override void Can_apply_two_migrations_in_transaction() => base.Can_apply_two_migrations_in_transaction(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_apply_two_migrations_in_transaction_async() => + base.Can_apply_two_migrations_in_transaction_async(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_idempotent_up_and_down_scripts() => + base.Can_generate_idempotent_up_and_down_scripts(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_idempotent_up_and_down_scripts_noTransactions() => + base.Can_generate_idempotent_up_and_down_scripts_noTransactions(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_one_up_and_down_script() => base.Can_generate_one_up_and_down_script(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_up_and_down_script_using_names() => + base.Can_generate_up_and_down_script_using_names(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_up_and_down_scripts() => base.Can_generate_up_and_down_scripts(); + + [ConditionalFact(Skip = "TODO")] + public override Task Can_generate_up_and_down_scripts_noTransactions() => + base.Can_generate_up_and_down_scripts_noTransactions(); + + [ConditionalFact(Skip = "TODO")] + public override void Can_revert_all_migrations() => base.Can_revert_all_migrations(); + + [ConditionalFact(Skip = "TODO")] + public override void Can_revert_one_migrations() => base.Can_revert_one_migrations(); + + public override void Can_get_active_provider() + { + base.Can_get_active_provider(); + + Assert.Equal("EfCore.Ydb", ActiveProvider); + } + + protected override Task ExecuteSqlAsync(string value) => + ((YdbTestStore)Fixture.TestStore).ExecuteNonQueryAsync(value); +} diff --git a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs index cca20e08..9976b9a7 100644 --- a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs +++ b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs @@ -23,7 +23,12 @@ public static YdbTestStore GetOrCreate( public override DbContextOptionsBuilder AddProviderOptions(DbContextOptionsBuilder builder) => UseConnectionString ? builder.UseYdb(Connection.ConnectionString) - : builder.UseYdb(Connection); + .LogTo(Console.WriteLine) + : builder.UseYdb(Connection) + .LogTo(Console.WriteLine); + + internal Task ExecuteNonQueryAsync(string sql, params object[] parameters) + => ExecuteAsync(Connection, command => command.ExecuteNonQueryAsync(), sql, false, parameters); protected override async Task InitializeAsync( Func createContext, @@ -162,10 +167,11 @@ private static YdbCommand CreateCommand( } - private static YdbConnection CreateConnection() => new(new YdbConnectionStringBuilder { MaxSessionPool = 10 }); + private static YdbConnection CreateConnection() => new(new YdbConnectionStringBuilder()); public override async Task CleanAsync(DbContext context) { + await context.Database.EnsureDeletedAsync(); var connection = context.Database.GetDbConnection(); if (connection.State != ConnectionState.Open) { diff --git a/src/Ydb.Sdk/src/Ado/YdbConnectionStringBuilder.cs b/src/Ydb.Sdk/src/Ado/YdbConnectionStringBuilder.cs index daa4bb5b..66a96619 100644 --- a/src/Ydb.Sdk/src/Ado/YdbConnectionStringBuilder.cs +++ b/src/Ydb.Sdk/src/Ado/YdbConnectionStringBuilder.cs @@ -275,7 +275,7 @@ private abstract class YdbConnectionOption return value switch { bool boolValue => boolValue, - string strValue => strValue switch + string strValue => strValue.ToLowerInvariant() switch { "on" or "true" or "1" => true, "off" or "false" or "0" => false,