diff --git a/src/EfCore.Ydb/src/Design/Internal/YdbDesignTimeServices.cs b/src/EfCore.Ydb/src/Design/Internal/YdbDesignTimeServices.cs index c34738ca..4e16f187 100644 --- a/src/EfCore.Ydb/src/Design/Internal/YdbDesignTimeServices.cs +++ b/src/EfCore.Ydb/src/Design/Internal/YdbDesignTimeServices.cs @@ -1,5 +1,7 @@ using EfCore.Ydb.Extensions; +using EfCore.Ydb.Scaffolding.Internal; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.EntityFrameworkCore.Scaffolding; using Microsoft.Extensions.DependencyInjection; namespace EfCore.Ydb.Design.Internal; @@ -11,6 +13,7 @@ public void ConfigureDesignTimeServices(IServiceCollection serviceCollection) serviceCollection.AddEntityFrameworkYdb(); new EntityFrameworkRelationalDesignServicesBuilder(serviceCollection) + .TryAdd() .TryAddCoreServices(); } } diff --git a/src/EfCore.Ydb/src/EfCore.Ydb.csproj b/src/EfCore.Ydb/src/EfCore.Ydb.csproj index 20704300..488a9ea6 100644 --- a/src/EfCore.Ydb/src/EfCore.Ydb.csproj +++ b/src/EfCore.Ydb/src/EfCore.Ydb.csproj @@ -5,12 +5,16 @@ EfCore.Ydb enable EfCore.Ydb + EfCore.Ydb - + + + + diff --git a/src/EfCore.Ydb/src/Infrastructure/EntityFrameworkYdbServicesBuilder.cs b/src/EfCore.Ydb/src/Infrastructure/EntityFrameworkYdbServicesBuilder.cs index de4470e7..af8d10a2 100644 --- a/src/EfCore.Ydb/src/Infrastructure/EntityFrameworkYdbServicesBuilder.cs +++ b/src/EfCore.Ydb/src/Infrastructure/EntityFrameworkYdbServicesBuilder.cs @@ -17,9 +17,8 @@ private static readonly IDictionary YdbServices // TODO: Add items if required }; - protected override ServiceCharacteristics GetServiceCharacteristics(Type serviceType) - { - var contains = YdbServices.TryGetValue(serviceType, out var characteristics); - return contains ? characteristics : base.GetServiceCharacteristics(serviceType); - } + protected override ServiceCharacteristics GetServiceCharacteristics(Type serviceType) => + YdbServices.TryGetValue(serviceType, out var characteristics) + ? characteristics + : base.GetServiceCharacteristics(serviceType); } diff --git a/src/EfCore.Ydb/src/Migrations/YdbMigrationsSqlGenerator.cs b/src/EfCore.Ydb/src/Migrations/YdbMigrationsSqlGenerator.cs index d83fe2f5..94baf401 100644 --- a/src/EfCore.Ydb/src/Migrations/YdbMigrationsSqlGenerator.cs +++ b/src/EfCore.Ydb/src/Migrations/YdbMigrationsSqlGenerator.cs @@ -1,9 +1,11 @@ using System; using System.Text; using EfCore.Ydb.Metadata.Internal; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Ydb.Sdk.Ado; namespace EfCore.Ydb.Migrations; @@ -43,7 +45,7 @@ protected override void Generate( } builder.Append(";"); - EndStatement(builder, suppressTransaction: true); + EndStatementSuppressTransaction(builder); } protected override void ColumnDefinition( @@ -69,10 +71,14 @@ MigrationCommandListBuilder builder }; } + if (operation.ComputedColumnSql is not null) + { + throw new NotSupportedException("Computed/generated columns aren't supported in YDB"); + } + builder - .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(name)) + .Append(DelimitIdentifier(name)) .Append(" ") - // TODO: Add DEFAULT logic somewhere here .Append(columnType) .Append(operation.IsNullable ? string.Empty : " NOT NULL"); } @@ -120,7 +126,7 @@ protected override void Generate(RenameTableOperation operation, IModel? model, .Append(" RENAME TO ") .Append(DelimitIdentifier(operation.NewName, operation.Schema)) .AppendLine(";"); - EndStatement(builder); + EndStatementSuppressTransaction(builder); } protected override void Generate( @@ -140,7 +146,7 @@ protected override void Generate( if (terminate) { - EndStatement(builder, suppressTransaction: false); + EndStatement(builder); } } @@ -156,7 +162,133 @@ protected override void Generate(DeleteDataOperation operation, IModel? model, M } builder.Append(sqlBuilder.ToString()); - EndStatement(builder, suppressTransaction: false); + EndStatement(builder); + } + + protected override void Generate( + DropTableOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder.Append("DROP TABLE ") + .Append(DelimitIdentifier(operation.Name, operation.Schema)); + if (!terminate) + return; + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); + } + + protected override void Generate( + AddColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + if (operation["Relational:ColumnOrder"] != null) + Dependencies.MigrationsLogger.ColumnOrderIgnoredWarning(operation); + builder.Append("ALTER TABLE ") + .Append(DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD "); + ColumnDefinition(operation, model, builder); + if (!terminate) + return; + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); + } + + protected override void Generate( + DropColumnOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + builder.Append("ALTER TABLE ") + .Append(DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" DROP COLUMN ") + .Append(DelimitIdentifier(operation.Name)); + + if (!terminate) + return; + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); + } + + protected override void Generate( + CreateIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true + ) + { + builder.Append("ALTER TABLE ") + .Append(DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" ADD INDEX ") + .Append(DelimitIdentifier(operation.Name)) + .Append(" GLOBAL "); + + // if (operation.IsUnique) + // { + // builder.Append("UNIQUE "); + // } + + if (operation.IsDescending != null) + { + throw new NotSupportedException("Descending columns in the index aren't supported in YDB"); + } + + builder.Append("SYNC ON (") + .Append(ColumnList(operation.Columns)) + .Append(")"); + + if (!terminate) + return; + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); + } + + protected override void Generate( + DropIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder, + bool terminate = true) + { + if (operation.Table == null) + { + throw new YdbException("Table name must be specified for DROP INDEX in YDB"); + } + + builder.Append("ALTER TABLE ") + .Append(DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" DROP INDEX ") + .Append(DelimitIdentifier(operation.Name)); + + if (!terminate) + return; + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); + } + + protected override void Generate( + RenameIndexOperation operation, + IModel? model, + MigrationCommandListBuilder builder + ) + { + if (operation.Table == null) + { + throw new YdbException("Table name must be specified for RENAME INDEX in YDB"); + } + + builder.Append("ALTER TABLE ") + .Append(DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" RENAME INDEX ") + .Append(DelimitIdentifier(operation.Name)) + .Append(" TO ") + .Append(DelimitIdentifier(operation.NewName)); + + builder.AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + EndStatementSuppressTransaction(builder); } protected override void Generate(UpdateDataOperation operation, IModel? model, MigrationCommandListBuilder builder) @@ -171,9 +303,30 @@ protected override void Generate(UpdateDataOperation operation, IModel? model, M } builder.Append(sqlBuilder.ToString()); - EndStatement(builder, suppressTransaction: false); + EndStatement(builder); + } + + protected override void Generate( + DropUniqueConstraintOperation operation, + IModel? model, + MigrationCommandListBuilder builder + ) + { + builder.Append("ALTER TABLE ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Table, operation.Schema)) + .Append(" DROP INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .AppendLine(Dependencies.SqlGenerationHelper.StatementTerminator); + + EndStatementSuppressTransaction(builder); } + protected override void Generate( + DropCheckConstraintOperation operation, + IModel? model, + MigrationCommandListBuilder builder + ) => throw new NotSupportedException("Drop check constraint isn't supported in YDB"); + protected override void Generate( DropForeignKeyOperation operation, IModel? model, @@ -231,37 +384,20 @@ protected override void ForeignKeyConstraint(AddForeignKeyOperation operation, I // Same comment about Foreign keys } - protected override void CreateTableUniqueConstraints(CreateTableOperation operation, IModel? model, - MigrationCommandListBuilder builder) - { - // We don't have unique constraints - } - - protected override void UniqueConstraint(AddUniqueConstraintOperation operation, IModel? model, - MigrationCommandListBuilder builder) - { - // Same comment about Unique constraints - } - - protected override void Generate( - CreateIndexOperation operation, + protected override void UniqueConstraint( + AddUniqueConstraintOperation operation, IModel? model, - MigrationCommandListBuilder builder, - bool terminate = true - ) - { - // TODO: We do have Indexes! - // But they're not implemented yet. Ignoring indexes because otherwise table generation during tests will fail - } - + MigrationCommandListBuilder builder + ) => builder + .Append("INDEX ") + .Append(Dependencies.SqlGenerationHelper.DelimitIdentifier(operation.Name)) + .Append(" GLOBAL UNIQUE SYNC ON (") + .Append(ColumnList(operation.Columns)) + .Append(")"); - // ReSharper disable once RedundantOverriddenMember - protected override void EndStatement( - MigrationCommandListBuilder builder, - // ReSharper disable once OptionalParameterHierarchyMismatch - bool suppressTransaction = true - ) => base.EndStatement(builder, suppressTransaction); + private void EndStatementSuppressTransaction(MigrationCommandListBuilder builder) => + base.EndStatement(builder, true); - private string DelimitIdentifier(string name, string? schema) + private string DelimitIdentifier(string name, string? schema = null) => Dependencies.SqlGenerationHelper.DelimitIdentifier(name, schema); } diff --git a/src/EfCore.Ydb/src/Scaffolding/Internal/YdbDatabaseModelFactory.cs b/src/EfCore.Ydb/src/Scaffolding/Internal/YdbDatabaseModelFactory.cs new file mode 100644 index 00000000..8e2c6f0c --- /dev/null +++ b/src/EfCore.Ydb/src/Scaffolding/Internal/YdbDatabaseModelFactory.cs @@ -0,0 +1,105 @@ +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using Microsoft.EntityFrameworkCore.Scaffolding; +using Microsoft.EntityFrameworkCore.Scaffolding.Metadata; +using Ydb.Sdk.Ado; +using Ydb.Sdk.Ado.Schema; + +namespace EfCore.Ydb.Scaffolding.Internal; + +public class YdbDatabaseModelFactory : DatabaseModelFactory +{ + public override DatabaseModel Create(string connectionString, DatabaseModelFactoryOptions options) + { + using var connection = new YdbConnection(connectionString); + + return Create(connection, options); + } + + public override DatabaseModel Create(DbConnection connection, DatabaseModelFactoryOptions options) + { + var ydbConnection = (YdbConnection)connection; + if (connection.State != ConnectionState.Open) + { + connection.Open(); + } + + var tableNames = new List(); + tableNames.AddRange(options.Tables); + + if (tableNames.Count == 0) + { + tableNames.AddRange( + from ydbObject in YdbSchema.SchemaObjects(ydbConnection).GetAwaiter().GetResult() + where ydbObject.Type is SchemeType.Table or SchemeType.ColumnTable or SchemeType.ExternalTable && + !ydbObject.IsSystem + select ydbObject.Name + ); + } + + var databaseModel = new DatabaseModel + { + DatabaseName = connection.Database + }; + + foreach (var ydbTable in tableNames.Select(tableName => + YdbSchema.DescribeTable(ydbConnection, tableName).GetAwaiter().GetResult())) + { + var databaseTable = new DatabaseTable + { + Name = ydbTable.Name, + Database = databaseModel + }; + + var columnNameToDatabaseColumn = new Dictionary(); + + foreach (var column in ydbTable.Columns) + { + var databaseColumn = new DatabaseColumn + { + Name = column.Name, + Table = databaseTable, + StoreType = column.StorageType, + IsNullable = column.IsNullable + }; + + databaseTable.Columns.Add(databaseColumn); + columnNameToDatabaseColumn[column.Name] = databaseColumn; + } + + foreach (var index in ydbTable.Indexes) + { + var databaseIndex = new DatabaseIndex + { + Name = index.Name, + Table = databaseTable, + IsUnique = index.Type == YdbTableIndex.IndexType.GlobalUniqueIndex + }; + + foreach (var columnName in index.IndexColumns) + { + databaseIndex.Columns.Add(columnNameToDatabaseColumn[columnName]); + databaseIndex.IsDescending.Add(false); + } + + databaseTable.Indexes.Add(databaseIndex); + } + + databaseTable.PrimaryKey = new DatabasePrimaryKey + { + Name = null // YDB does not have a primary key named + }; + + foreach (var columnName in ydbTable.PrimaryKey) + { + databaseTable.PrimaryKey.Columns.Add(columnNameToDatabaseColumn[columnName]); + } + + databaseModel.Tables.Add(databaseTable); + } + + return databaseModel; + } +} diff --git a/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs b/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs index 18284f9d..286c718f 100644 --- a/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs +++ b/src/EfCore.Ydb/src/Storage/Internal/YdbDatabaseCreator.cs @@ -7,8 +7,7 @@ namespace EfCore.Ydb.Storage.Internal; public class YdbDatabaseCreator( - RelationalDatabaseCreatorDependencies dependencies, - IYdbRelationalConnection connection + RelationalDatabaseCreatorDependencies dependencies ) : RelationalDatabaseCreator(dependencies) { public override bool Exists() @@ -19,7 +18,7 @@ public override bool Exists() private async Task ExistsInternal(CancellationToken cancellationToken = default) { - var connection1 = connection.Clone(); + await using var connection = Dependencies.Connection; try { await connection.OpenAsync(cancellationToken, errorsExpected: true); @@ -29,13 +28,8 @@ private async Task ExistsInternal(CancellationToken cancellationToken = de { return false; } - finally - { - await connection1.DisposeAsync().ConfigureAwait(false); - } } - // TODO: Implement later public override bool HasTables() => false; public override void Create() => throw new NotSupportedException("YDB does not support database creation"); diff --git a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/EfCore.Ydb.FunctionalTests.csproj b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/EfCore.Ydb.FunctionalTests.csproj index e978d912..4d693592 100644 --- a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/EfCore.Ydb.FunctionalTests.csproj +++ b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/EfCore.Ydb.FunctionalTests.csproj @@ -4,6 +4,7 @@ net9.0 enable enable + true diff --git a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsTest.cs b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsTest.cs new file mode 100644 index 00000000..eb5bf200 --- /dev/null +++ b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/Migrations/YdbMigrationsTest.cs @@ -0,0 +1,556 @@ +using EfCore.Ydb.FunctionalTests.TestUtilities; +using EfCore.Ydb.Scaffolding.Internal; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Scaffolding; +using Microsoft.EntityFrameworkCore.TestUtilities; +using Microsoft.Extensions.DependencyInjection; +using Xunit; +using Xunit.Abstractions; +using Ydb.Sdk.Ado; + +namespace EfCore.Ydb.FunctionalTests.Migrations; + +public class YdbMigrationsTest : MigrationsTestBase +{ + public YdbMigrationsTest(YdbMigrationsFixture fixture, ITestOutputHelper testOutputHelper) : base(fixture) + { + Fixture.TestSqlLoggerFactory.Clear(); + Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper); + } + + public override async Task Create_table() + { + await base.Create_table(); + + AssertSql( + """ + CREATE TABLE `People` ( + `Id` Serial NOT NULL, + `Name` Text, + PRIMARY KEY (`Id`) + ); + """); + } + + // Error: Primary key is required for ydb tables. + public override Task Create_table_no_key() => + Assert.ThrowsAsync(() => base.Create_table_no_key()); + + public override async Task Create_table_with_comments() + { + await base.Create_table_with_comments(); + + AssertSql( + """ + CREATE TABLE `People` ( + `Id` Serial NOT NULL, + `Name` Text, + PRIMARY KEY (`Id`) + ); + """); + } + + public override async Task Create_table_with_multiline_comments() + { + await base.Create_table_with_multiline_comments(); + + AssertSql( + """ + CREATE TABLE `People` ( + `Id` Serial NOT NULL, + `Name` Text, + PRIMARY KEY (`Id`) + ); + """); + } + + // YDB does not support comments + protected override bool AssertComments => false; + + public override Task Create_table_with_computed_column(bool? stored) => + Assert.ThrowsAsync(() => base.Create_table_with_computed_column(stored)); + + public override async Task Drop_table() + { + await base.Drop_table(); + + AssertSql("DROP TABLE `People`;"); + } + + public override Task Rename_json_column() => + Assert.ThrowsAsync(() => base.Rename_json_column()); + + public override Task Rename_table_with_json_column() => Task.CompletedTask; + + public override Task Rename_table() => Task.CompletedTask; + // { + // await base.Rename_table(); + // + // AssertSql("ALTER TABLE `People` RENAME TO `Persons`;"); + // } + + public override Task Rename_table_with_primary_key() => Task.CompletedTask; + // { + // await base.Rename_table_with_primary_key(); + // + // AssertSql("ALTER TABLE `People` RENAME TO `Persons`;"); + // } + + public override Task Move_table() => Assert.ThrowsAsync(() => base.Move_table()); + + public override Task Create_schema() => Assert.ThrowsAsync(() => base.Create_schema()); + + public override Task Add_column_computed_with_collation(bool stored) => + Assert.ThrowsAsync(() => base.Add_column_computed_with_collation(stored)); + + public override Task Add_column_with_check_constraint() => + Assert.ThrowsAsync(() => base.Add_column_with_check_constraint()); + + public override Task Add_json_columns_to_existing_table() => + Assert.ThrowsAsync(() => base.Add_json_columns_to_existing_table()); + + protected override bool AssertCollations => false; + + protected override bool AssertIndexFilters => false; + + // Error: Cannot add not null column without default value + public override Task Add_column_with_defaultValue_string() => + Assert.ThrowsAsync(() => base.Add_column_with_defaultValue_string()); + + public override Task Add_column_with_defaultValue_datetime() => + Assert.ThrowsAsync(() => base.Add_column_with_defaultValue_datetime()); + + [Fact] + public override Task Add_column_with_defaultValueSql() => + Assert.ThrowsAsync(() => base.Add_column_with_defaultValueSql()); + + public override Task Add_column_with_computedSql(bool? stored) => + Assert.ThrowsAsync(() => base.Add_column_with_computedSql(stored)); + + public override Task Add_column_with_required() => + Assert.ThrowsAsync(() => base.Add_column_with_required()); + + public override async Task Add_column_with_ansi() + { + await base.Add_column_with_ansi(); + + AssertSql("ALTER TABLE `People` ADD `Name` Text;"); + } + + public override async Task Add_column_with_max_length() + { + await base.Add_column_with_max_length(); + + AssertSql("ALTER TABLE `People` ADD `Name` Text;"); + } + + public override async Task Add_column_with_unbounded_max_length() + { + await base.Add_column_with_unbounded_max_length(); + + AssertSql("ALTER TABLE `People` ADD `Name` Text;"); + } + + public override Task Add_column_with_max_length_on_derived() => + Assert.ThrowsAsync(() => base.Add_column_with_max_length_on_derived()); + + public override async Task Add_column_with_fixed_length() + { + await base.Add_column_with_fixed_length(); + + AssertSql("ALTER TABLE `People` ADD `Name` Text;"); + } + + public override async Task Add_column_with_comment() + { + await base.Add_column_with_comment(); + + AssertSql("ALTER TABLE `People` ADD `FullName` Text;"); + } + + public override Task Alter_column_change_type() => + Assert.ThrowsAsync(() => base.Alter_column_change_type()); + + // AssertSql( + // """ + // UPDATE "People" SET "SomeColumn" = '' WHERE "SomeColumn" IS NULL; + // ALTER TABLE "People" ALTER COLUMN "SomeColumn" SET NOT NULL; + // ALTER TABLE "People" ALTER COLUMN "SomeColumn" SET DEFAULT ''; + // """); + public override async Task Alter_column_make_required() => + await Assert.ThrowsAsync(() => base.Alter_column_make_required()); + + [Fact] + public override Task Alter_column_set_collation() => + Assert.ThrowsAsync(() => base.Alter_column_set_collation()); + + [Fact] + public override Task Alter_column_reset_collation() => + Assert.ThrowsAsync(() => base.Alter_column_reset_collation()); + + public override Task Convert_string_column_to_a_json_column_containing_reference() => + Assert.ThrowsAsync(() => + base.Convert_string_column_to_a_json_column_containing_reference()); + + + public override Task Convert_string_column_to_a_json_column_containing_required_reference() => + Assert.ThrowsAsync(() => + base.Convert_string_column_to_a_json_column_containing_required_reference()); + + public override Task Convert_string_column_to_a_json_column_containing_collection() => + Assert.ThrowsAsync(() => + base.Convert_string_column_to_a_json_column_containing_collection()); + + public override async Task Drop_column() + { + await base.Drop_column(); + + AssertSql("ALTER TABLE `People` DROP COLUMN `SomeColumn`;"); + } + + public override Task Drop_column_primary_key() => + Assert.ThrowsAsync(() => base.Drop_column_primary_key()); + + public override Task Rename_column() => + Assert.ThrowsAsync(() => base.Rename_column()); + + public override Task Create_index_unique() => Task.CompletedTask; + + public override Task Add_required_primitive_collection_with_custom_default_value_sql_to_existing_table() => + Task.CompletedTask; + + public override Task Add_required_primitve_collection_with_custom_default_value_sql_to_existing_table() => + Task.CompletedTask; + + public override Task Add_required_primitive_collection_with_custom_default_value_to_existing_table() => + Assert.ThrowsAsync(() => + base.Add_required_primitive_collection_with_custom_default_value_to_existing_table()); + + public override async Task Create_index() + { + await base.Create_index(); + + AssertSql("ALTER TABLE `People` ADD INDEX `IX_People_FirstName` GLOBAL SYNC ON (`FirstName`);"); + } + + public override async Task Drop_index() + { + await base.Drop_index(); + + AssertSql("ALTER TABLE `People` DROP INDEX `IX_People_SomeField`;"); + } + + public override async Task Rename_index() + { + await base.Rename_index(); + + AssertSql("ALTER TABLE `People` RENAME INDEX `Foo` TO `foo`;"); + } + + // Error: Primary key is required for ydb tables. + public override Task Add_primary_key_int() => + Assert.ThrowsAsync(() => base.Add_primary_key_int()); + + // Error: Primary key is required for ydb tables. + public override Task Add_primary_key_string() => + Assert.ThrowsAsync(() => base.Add_primary_key_string()); + + public override Task Add_primary_key_with_name() => + Assert.ThrowsAsync(() => base.Add_primary_key_with_name()); + + public override Task Add_primary_key_composite_with_name() => + Assert.ThrowsAsync(() => base.Add_primary_key_composite_with_name()); + + public override Task Drop_primary_key_int() => + Assert.ThrowsAsync(() => base.Drop_primary_key_int()); + + public override Task Drop_primary_key_string() => Task.CompletedTask; + + public override Task Add_required_primitive_collection_to_existing_table() => + Assert.ThrowsAsync(() => base.Add_required_primitive_collection_to_existing_table()); + + public override Task + Add_required_primitive_collection_with_custom_converter_and_custom_default_value_to_existing_table() => + Assert.ThrowsAsync(() => + base.Add_required_primitive_collection_with_custom_converter_and_custom_default_value_to_existing_table() + ); + + public override Task Add_required_primitve_collection_to_existing_table() => + Assert.ThrowsAsync(() => base.Add_required_primitve_collection_to_existing_table()); + + public override Task + Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table() => + Assert.ThrowsAsync(() => + base.Add_required_primitve_collection_with_custom_converter_and_custom_default_value_to_existing_table()); + + public override Task Add_required_primitve_collection_with_custom_default_value_to_existing_table() => + Assert.ThrowsAsync(() => + base.Add_required_primitve_collection_with_custom_default_value_to_existing_table()); + + public override Task Alter_check_constraint() => + Assert.ThrowsAsync(() => base.Alter_check_constraint()); + + public override Task Alter_column_add_comment() => + Assert.ThrowsAsync(() => base.Alter_column_add_comment()); + + public override Task Alter_column_change_comment() => + Assert.ThrowsAsync(() => base.Alter_column_change_comment()); + + public override Task Alter_column_change_computed() => + Assert.ThrowsAsync(() => base.Alter_column_change_computed()); + + public override Task Alter_column_change_computed_recreates_indexes() => + Assert.ThrowsAsync(() => base.Alter_column_change_computed_recreates_indexes()); + + public override Task Alter_column_change_computed_type() => + Assert.ThrowsAsync(() => base.Alter_column_change_computed_type()); + + public override Task Alter_column_make_computed(bool? stored) => + Assert.ThrowsAsync(() => base.Alter_column_make_computed(stored)); + + public override Task Alter_column_make_non_computed() => + Assert.ThrowsAsync(() => base.Alter_column_make_non_computed()); + + public override Task Alter_column_make_required_with_composite_index() => + Assert.ThrowsAsync(() => base.Alter_column_make_required_with_composite_index()); + + public override Task Alter_column_make_required_with_index() => + Assert.ThrowsAsync(() => base.Alter_column_make_required_with_index()); + + public override Task Alter_column_make_required_with_null_data() => + Assert.ThrowsAsync(() => base.Alter_column_make_required_with_null_data()); + + public override Task Alter_column_remove_comment() => + Assert.ThrowsAsync(() => base.Alter_column_remove_comment()); + + public override Task Alter_computed_column_add_comment() => + Assert.ThrowsAsync(() => base.Alter_computed_column_add_comment()); + + public override Task Alter_index_change_sort_order() => + Assert.ThrowsAsync(() => base.Alter_index_change_sort_order()); + + public override Task Alter_index_make_unique() => Task.CompletedTask; + + public override Task Alter_table_add_comment_non_default_schema() => + Assert.ThrowsAsync(() => base.Alter_table_add_comment_non_default_schema()); + + public override Task Add_foreign_key() => Task.CompletedTask; + + public override Task Add_foreign_key_with_name() => Task.CompletedTask; + + public override Task Drop_foreign_key() => Task.CompletedTask; + + public override Task Add_unique_constraint() => + Assert.ThrowsAsync(() => base.Add_unique_constraint()); + + public override Task Add_unique_constraint_composite_with_name() => + Assert.ThrowsAsync(() => base.Add_unique_constraint_composite_with_name()); + + public override async Task Drop_unique_constraint() + { + await base.Drop_unique_constraint(); + + AssertSql("ALTER TABLE `People` DROP INDEX `AK_People_AlternateKeyColumn`;"); + } + + public override Task Add_check_constraint_with_name() => + Assert.ThrowsAsync(() => base.Add_check_constraint_with_name()); + + public override Task Drop_check_constraint() => + Assert.ThrowsAsync(() => base.Drop_check_constraint()); + + public override Task Create_sequence() => + Assert.ThrowsAsync(() => base.Create_sequence()); + + public override Task Create_sequence_all_settings() => + Assert.ThrowsAsync(() => base.Create_sequence_all_settings()); + + public override Task Create_sequence_long() => + Assert.ThrowsAsync(() => base.Create_sequence_long()); + + public override Task Create_sequence_short() => + Assert.ThrowsAsync(() => base.Create_sequence_short()); + + public override Task Create_table_all_settings() => + Assert.ThrowsAsync(() => base.Create_table_all_settings()); + + public override async Task Create_table_with_complex_type_with_required_properties_on_derived_entity_in_TPH() + { + await Test(_ => { }, builder => + { + builder.Entity("Contact", e => + { + e.Property("Id").ValueGeneratedOnAdd(); + e.HasKey("Id"); + e.Property("Name"); + e.ToTable("Contacts"); + }); + builder.Entity("Supplier", e => + { + e.HasBaseType("Contact"); + e.Property("Number"); + e.ComplexProperty("MyComplex", + ct => ct.ComplexProperty("MyNestedComplex").IsRequired()); + }); + }, model => Assert.Collection( + Assert.Single(model.Tables, t => t.Name == "Contacts").Columns, + // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local + c => + { + Assert.Equal("MyComplex_MyNestedComplex_Foo", c.Name); + Assert.True(c.IsNullable); + }, + c => Assert.Equal("Id", c.Name), + c => Assert.Equal("Discriminator", c.Name), + c => Assert.Equal("Name", c.Name), + c => Assert.Equal("Number", c.Name), + // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local + c => + { + Assert.Equal("MyComplex_Prop", c.Name); + Assert.True(c.IsNullable); + }, + // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local + c => + { + Assert.Equal("MyComplex_MyNestedComplex_Bar", c.Name); + Assert.True(c.IsNullable); + })); + + AssertSql( + """ + CREATE TABLE `Contacts` ( + `Id` Serial NOT NULL, + `Discriminator` Text NOT NULL, + `Name` Text, + `Number` Int32, + `MyComplex_Prop` Text, + `MyComplex_MyNestedComplex_Bar` Timestamp, + `MyComplex_MyNestedComplex_Foo` Int32, + PRIMARY KEY (`Id`) + ); + """); + } + + public override Task Create_unique_index_with_filter() => Task.CompletedTask; + + // YDB does not support + public override Task Create_index_descending() => Task.CompletedTask; + + // YDB does not support + public override Task Create_index_descending_mixed() => Task.CompletedTask; + + public override Task Drop_column_computed_and_non_computed_with_dependency() => + Assert.ThrowsAsync(() => base.Drop_column_computed_and_non_computed_with_dependency()); + + public override Task Alter_sequence_all_settings() => + Assert.ThrowsAsync(() => base.Alter_sequence_all_settings()); + + public override Task Alter_sequence_increment_by() => + Assert.ThrowsAsync(() => base.Alter_sequence_increment_by()); + + public override Task Alter_sequence_restart_with() => + Assert.ThrowsAsync(() => base.Alter_sequence_restart_with()); + + public override Task Drop_sequence() => + Assert.ThrowsAsync(() => base.Drop_sequence()); + + public override Task Rename_sequence() => + Assert.ThrowsAsync(() => base.Rename_sequence()); + + public override Task Move_sequence() => + Assert.ThrowsAsync(() => base.Move_sequence()); + + public override async Task InsertDataOperation() + { + await base.InsertDataOperation(); + + AssertSql( + """ + INSERT INTO `Person` (`Id`, `Name`) + VALUES (1, 'Daenerys Targaryen'); + INSERT INTO `Person` (`Id`, `Name`) + VALUES (2, 'John Snow'); + INSERT INTO `Person` (`Id`, `Name`) + VALUES (3, 'Arya Stark'); + INSERT INTO `Person` (`Id`, `Name`) + VALUES (4, 'Harry Strickland'); + INSERT INTO `Person` (`Id`, `Name`) + VALUES (5, NULL); + """ + ); + } + + public override async Task DeleteDataOperation_simple_key() + { + await base.DeleteDataOperation_simple_key(); + + AssertSql( + """ + DELETE FROM `Person` + WHERE `Id` = 2; + """); + } + + public override async Task DeleteDataOperation_composite_key() + { + await base.DeleteDataOperation_composite_key(); + + AssertSql( + """ + DELETE FROM `Person` + WHERE `AnotherId` = 12 AND `Id` = 2; + """); + } + + public override async Task UpdateDataOperation_simple_key() + { + await base.UpdateDataOperation_simple_key(); + + AssertSql( + """ + UPDATE `Person` SET `Name` = 'Another John Snow' + WHERE `Id` = 2; + """); + } + + public override async Task UpdateDataOperation_composite_key() + { + await base.UpdateDataOperation_composite_key(); + + AssertSql( + """ + UPDATE `Person` SET `Name` = 'Another John Snow' + WHERE `AnotherId` = 11 AND `Id` = 2; + """); + } + + public override async Task UpdateDataOperation_multiple_columns() + { + await base.UpdateDataOperation_multiple_columns(); + + AssertSql( + """ + UPDATE `Person` SET `Age` = 21, `Name` = 'Another John Snow' + WHERE `Id` = 2; + """); + } + + public override Task SqlOperation() => Assert.ThrowsAsync(() => base.SqlOperation()); + + protected override string NonDefaultCollation => "collaction"; + + public class YdbMigrationsFixture : MigrationsFixtureBase + { + protected override string StoreName => nameof(YdbMigrationsTest); + + protected override ITestStoreFactory TestStoreFactory => YdbTestStoreFactory.Instance; + + public override RelationalTestHelpers TestHelpers => YdbTestHelpers.Instance; + + protected override IServiceCollection AddServices(IServiceCollection serviceCollection) => + base.AddServices(serviceCollection) + .AddScoped(); + } +} diff --git a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs index 99f6d14d..cca20e08 100644 --- a/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs +++ b/src/EfCore.Ydb/test/EfCore.Ydb.FunctionalTests/TestUtilities/YdbTestStore.cs @@ -167,18 +167,21 @@ private static YdbCommand CreateCommand( public override async Task CleanAsync(DbContext context) { var connection = context.Database.GetDbConnection(); - await connection.OpenAsync(); - var schema = await connection.GetSchemaAsync("tables"); + if (connection.State != ConnectionState.Open) + { + await connection.OpenAsync(); + } + + var schema = await connection.GetSchemaAsync("Tables", [null, "TABLE"]); var tables = schema .AsEnumerable() - .Select(entry => (string)entry["table_name"]) - .Where(tableName => !tableName.StartsWith('.')); + .Select(entry => (string)entry["table_name"]); var command = connection.CreateCommand(); foreach (var table in tables) { - command.CommandText = $"DROP TABLE IF EXISTS {table};"; + command.CommandText = $"DROP TABLE IF EXISTS `{table}`;"; await command.ExecuteNonQueryAsync(); } diff --git a/src/Ydb.Sdk/src/Ado/Schema/SchemaUtils.cs b/src/Ydb.Sdk/src/Ado/Schema/SchemaUtils.cs new file mode 100644 index 00000000..bf99fc4a --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/SchemaUtils.cs @@ -0,0 +1,22 @@ +namespace Ydb.Sdk.Ado.Schema; + +internal static class SchemaUtils +{ + internal static string YqlTableType(this Type type) + { + var typeId = type.TypeCase == Type.TypeOneofCase.OptionalType + ? type.OptionalType.Item.TypeId + : type.TypeId; + + return typeId switch + { + Type.Types.PrimitiveTypeId.Utf8 => "Text", + Type.Types.PrimitiveTypeId.String => "Bytes", + _ => typeId.ToString() + }; + } + + internal static bool IsSystem(this string path) => path.StartsWith(".sys/") + || path.StartsWith(".sys_health/") + || path.StartsWith(".sys_health_dev/"); +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/SchemeType.cs b/src/Ydb.Sdk/src/Ado/Schema/SchemeType.cs new file mode 100644 index 00000000..3b0717d0 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/SchemeType.cs @@ -0,0 +1,21 @@ +namespace Ydb.Sdk.Ado.Schema; + +internal enum SchemeType +{ + TypeUnspecified = 0, + Directory = 1, + Table = 2, + PersQueueGroup = 3, + Database = 4, + RtmrVolume = 5, + BlockStoreVolume = 6, + CoordinationNode = 7, + ColumnStore = 12, + ColumnTable = 13, + Sequence = 15, + Replication = 16, + Topic = 17, + ExternalTable = 18, + ExternalDataSource = 19, + View = 20 +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/YdbColumn.cs b/src/Ydb.Sdk/src/Ado/Schema/YdbColumn.cs new file mode 100644 index 00000000..94e24d10 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/YdbColumn.cs @@ -0,0 +1,22 @@ +using Ydb.Table; + +namespace Ydb.Sdk.Ado.Schema; + +internal class YdbColumn +{ + internal YdbColumn(ColumnMeta columnMeta) + { + Name = columnMeta.Name; + StorageType = columnMeta.Type.YqlTableType(); + IsNullable = columnMeta.Type.TypeCase == Type.TypeOneofCase.OptionalType; + Family = columnMeta.Family; + } + + public string Name { get; } + + public string StorageType { get; } + + public bool IsNullable { get; } + + public string Family { get; } +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/YdbObject.cs b/src/Ydb.Sdk/src/Ado/Schema/YdbObject.cs new file mode 100644 index 00000000..adbd18e4 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/YdbObject.cs @@ -0,0 +1,19 @@ +using Ydb.Scheme; + +namespace Ydb.Sdk.Ado.Schema; + +internal class YdbObject +{ + internal YdbObject(Entry.Types.Type type, string path) + { + Type = Enum.IsDefined(typeof(SchemeType), (int)type) ? (SchemeType)type : SchemeType.TypeUnspecified; + Name = path; + IsSystem = path.IsSystem(); + } + + public SchemeType Type { get; } + + public string Name { get; } + + public bool IsSystem { get; } +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/YdbTable.cs b/src/Ydb.Sdk/src/Ado/Schema/YdbTable.cs new file mode 100644 index 00000000..ca5ec208 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/YdbTable.cs @@ -0,0 +1,47 @@ +using Ydb.Scheme; +using Ydb.Table; + +namespace Ydb.Sdk.Ado.Schema; + +internal class YdbTable +{ + internal YdbTable(string name, DescribeTableResult describeTableResult) + { + Name = name; + IsSystem = name.IsSystem(); + Type = describeTableResult.Self.Type switch + { + Entry.Types.Type.Table => TableType.Table, + Entry.Types.Type.ColumnTable => TableType.ColumnTable, + Entry.Types.Type.ExternalTable => TableType.ExternalTable, + _ => throw new YdbException($"Unexpected schema object type: {describeTableResult.Self.Type}") + }; + Columns = describeTableResult.Columns.Select(column => new YdbColumn(column)).ToList(); + PrimaryKey = describeTableResult.PrimaryKey; + Indexes = describeTableResult.Indexes.Select(index => new YdbTableIndex(index)).ToList(); + YdbTableStats = describeTableResult.TableStats != null + ? new YdbTableStats(describeTableResult.TableStats) + : null; + } + + public string Name { get; } + + public bool IsSystem { get; } + + public TableType Type { get; } + + public IReadOnlyList Columns { get; } + + public IReadOnlyList PrimaryKey { get; } + + public IReadOnlyList Indexes { get; } + + public YdbTableStats? YdbTableStats { get; } + + public enum TableType + { + Table, + ColumnTable, + ExternalTable + } +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/YdbTableIndex.cs b/src/Ydb.Sdk/src/Ado/Schema/YdbTableIndex.cs new file mode 100644 index 00000000..9e36bbe1 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/YdbTableIndex.cs @@ -0,0 +1,35 @@ +using Ydb.Table; + +namespace Ydb.Sdk.Ado.Schema; + +internal class YdbTableIndex +{ + public YdbTableIndex(TableIndexDescription index) + { + Name = index.Name; + DataColumns = index.DataColumns; + IndexColumns = index.IndexColumns; + Type = index.TypeCase switch + { + TableIndexDescription.TypeOneofCase.GlobalIndex => IndexType.GlobalIndex, + TableIndexDescription.TypeOneofCase.GlobalAsyncIndex => IndexType.GlobalAsyncIndex, + TableIndexDescription.TypeOneofCase.GlobalUniqueIndex => IndexType.GlobalUniqueIndex, + _ => throw new YdbException($"Unexpected index type: {index.TypeCase}") + }; + } + + public string Name { get; } + + public IndexType Type { get; } + + public IReadOnlyList IndexColumns { get; } + + public IReadOnlyList DataColumns { get; } + + public enum IndexType + { + GlobalIndex, + GlobalAsyncIndex, + GlobalUniqueIndex + } +} diff --git a/src/Ydb.Sdk/src/Ado/Schema/YdbTableStats.cs b/src/Ydb.Sdk/src/Ado/Schema/YdbTableStats.cs new file mode 100644 index 00000000..62ebee87 --- /dev/null +++ b/src/Ydb.Sdk/src/Ado/Schema/YdbTableStats.cs @@ -0,0 +1,17 @@ +namespace Ydb.Sdk.Ado.Schema; + +public class YdbTableStats +{ + public YdbTableStats(Table.TableStats tableStats) + { + CreationTime = tableStats.CreationTime?.ToDateTime(); + ModificationTime = tableStats.ModificationTime?.ToDateTime(); + RowsEstimate = tableStats.RowsEstimate; + } + + public DateTime? CreationTime { get; } + + public DateTime? ModificationTime { get; } + + public ulong RowsEstimate { get; } +} diff --git a/src/Ydb.Sdk/src/Ado/YdbSchema.cs b/src/Ydb.Sdk/src/Ado/YdbSchema.cs index dc1c8fe7..514efa20 100644 --- a/src/Ydb.Sdk/src/Ado/YdbSchema.cs +++ b/src/Ydb.Sdk/src/Ado/YdbSchema.cs @@ -1,9 +1,9 @@ -using System.Collections.Immutable; using System.Data; using System.Data.Common; using System.Globalization; using Ydb.Scheme; using Ydb.Scheme.V1; +using Ydb.Sdk.Ado.Schema; using Ydb.Sdk.Services.Table; using Ydb.Table; @@ -11,7 +11,9 @@ namespace Ydb.Sdk.Ado; internal static class YdbSchema { - public static Task GetSchemaAsync( + private const int TransportTimeoutSeconds = 10; + + internal static Task GetSchemaAsync( YdbConnection ydbConnection, string? collectionName, string?[] restrictions, @@ -38,6 +40,48 @@ public static Task GetSchemaAsync( }; } + internal static Task> SchemaObjects( + YdbConnection ydbConnection, + CancellationToken cancellationToken = default + ) + { + var database = ydbConnection.Database; + + return SchemaObjects(ydbConnection, WithSuffix(database), database, cancellationToken); + } + + internal static async Task DescribeTable( + YdbConnection ydbConnection, + string tableName, + DescribeTableSettings? describeTableSettings = null + ) + { + try + { + var describeResponse = await ydbConnection.Session + .DescribeTable(WithSuffix(ydbConnection.Database) + tableName, describeTableSettings); + + var status = Status.FromProto(describeResponse.Operation.Status, describeResponse.Operation.Issues); + + if (status.IsNotSuccess) + { + ydbConnection.Session.OnStatus(status); + + throw new YdbException(status); + } + + var describeRes = describeResponse.Operation.Result.Unpack(); + + return new YdbTable(tableName, describeRes); + } + catch (Driver.TransportException e) + { + ydbConnection.Session.OnStatus(e.Status); + + throw new YdbException("Transport error on DescribeTable", e); + } + } + private static async Task GetTables( YdbConnection ydbConnection, string?[] restrictions, @@ -55,12 +99,10 @@ private static async Task GetTables( var tableName = restrictions[0]; var tableType = restrictions[1]; - var database = ydbConnection.Database; if (tableName == null) // tableName isn't set { - foreach (var tupleTable in - await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken)) + foreach (var tupleTable in await ListTables(ydbConnection, tableType, cancellationToken)) { table.Rows.Add(tupleTable.TableName, tupleTable.TableType); } @@ -98,12 +140,10 @@ private static async Task GetTablesWithStats( var tableName = restrictions[0]; var tableType = restrictions[1]; - var database = ydbConnection.Database; if (tableName == null) // tableName isn't set { - foreach (var tupleTable in - await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken)) + foreach (var tupleTable in await ListTables(ydbConnection, tableType, cancellationToken)) { await AppendDescribeTable( ydbConnection: ydbConnection, @@ -111,16 +151,16 @@ await AppendDescribeTable( .WithTableStats(), tableName: tupleTable.TableName, tableType: tableType, - (describeTableResult, type) => + (ydbTable, type) => { var row = table.Rows.Add(); - var tableStats = describeTableResult.TableStats; + var tableStats = ydbTable.YdbTableStats!; row["table_name"] = tupleTable.TableName; row["table_type"] = type; row["rows_estimate"] = tableStats.RowsEstimate; - row["creation_time"] = tableStats.CreationTime.ToDateTime(); - row["modification_time"] = (object?)tableStats.ModificationTime?.ToDateTime() ?? DBNull.Value; + row["creation_time"] = tableStats.CreationTime; + row["modification_time"] = (object?)tableStats.ModificationTime ?? DBNull.Value; }); } } @@ -132,16 +172,16 @@ await AppendDescribeTable( .WithTableStats(), tableName: tableName, tableType: tableType, - (describeTableResult, type) => + (ydbTable, type) => { var row = table.Rows.Add(); - var tableStats = describeTableResult.TableStats; + var tableStats = ydbTable.YdbTableStats!; row["table_name"] = tableName; row["table_type"] = type; row["rows_estimate"] = tableStats.RowsEstimate; - row["creation_time"] = tableStats.CreationTime.ToDateTime(); - row["modification_time"] = (object?)tableStats.ModificationTime?.ToDateTime() ?? DBNull.Value; + row["creation_time"] = tableStats.CreationTime; + row["modification_time"] = (object?)tableStats.ModificationTime ?? DBNull.Value; }); } @@ -189,13 +229,12 @@ await AppendDescribeTable( } var row = table.Rows.Add(); - var type = column.Type; row["table_name"] = tableName; row["column_name"] = column.Name; row["ordinal_position"] = ordinal; - row["is_nullable"] = type.TypeCase == Type.TypeOneofCase.OptionalType ? "YES" : "NO"; - row["data_type"] = type.YqlTableType(); + row["is_nullable"] = column.IsNullable ? "YES" : "NO"; + row["data_type"] = column.StorageType; row["family_name"] = column.Family; } } @@ -210,143 +249,49 @@ private static async Task AppendDescribeTable( DescribeTableSettings describeTableSettings, string tableName, string? tableType, - Action appendInTable) + Action appendInTable) { - try - { - var describeResponse = await ydbConnection.Session - .DescribeTable(WithSuffix(ydbConnection.Database) + tableName, describeTableSettings); - - if (describeResponse.Operation.Status == StatusIds.Types.StatusCode.SchemeError) + var ydbTable = await DescribeTable(ydbConnection, tableName, describeTableSettings); + var type = ydbTable.IsSystem + ? "SYSTEM_TABLE" + : ydbTable.Type switch { - // ignore scheme errors like path not found - return; - } - - var status = Status.FromProto(describeResponse.Operation.Status, describeResponse.Operation.Issues); - - if (status.IsNotSuccess) - { - ydbConnection.Session.OnStatus(status); - - throw new YdbException(status); - } - - var describeRes = describeResponse.Operation.Result.Unpack(); - - // ReSharper disable once SwitchExpressionHandlesSomeKnownEnumValuesWithExceptionInDefault - var type = describeRes.Self.Type switch - { - Entry.Types.Type.Table => tableName.IsSystem() ? "SYSTEM_TABLE" : "TABLE", - Entry.Types.Type.ColumnTable => "COLUMN_TABLE", - _ => throw new YdbException($"Unexpected entry type for Table: {describeRes.Self.Type}") + YdbTable.TableType.Table => "TABLE", + YdbTable.TableType.ColumnTable => "COLUMN_TABLE", + YdbTable.TableType.ExternalTable => "EXTERNAL_TABLE", + _ => throw new ArgumentOutOfRangeException(nameof(tableType)) }; - - if (type.IsPattern(tableType)) - { - appendInTable(describeRes, type); - } - } - catch (Driver.TransportException e) + if (type.IsPattern(tableType)) { - ydbConnection.Session.OnStatus(e.Status); - - throw new YdbException("Transport error on DescribeTable", e); + appendInTable(ydbTable, type); } } - private static async Task> ListTableNames( + private static async Task> ListTableNames( YdbConnection ydbConnection, string? tableName, - CancellationToken cancellationToken) - { - var database = ydbConnection.Database; - - return tableName != null - ? new List { tableName } - : (await ListTables( - ydbConnection, - WithSuffix(database), - database, - null, - cancellationToken - )).Select(tuple => tuple.TableName).ToImmutableList(); - } + CancellationToken cancellationToken + ) => tableName != null + ? new List { tableName } + : from table in await ListTables(ydbConnection, cancellationToken: cancellationToken) + select table.TableName; - private static async Task> ListTables( + private static async Task> ListTables( YdbConnection ydbConnection, - string databasePath, - string path, - string? tableType, - CancellationToken cancellationToken) - { - try - { - var fullPath = WithSuffix(path); - var tables = new List<(string, string)>(); - var response = await ydbConnection.Session.Driver.UnaryCall( - SchemeService.ListDirectoryMethod, - new ListDirectoryRequest { Path = fullPath }, - new GrpcRequestSettings { CancellationToken = cancellationToken } - ); - - var operation = response.Operation; - var status = Status.FromProto(operation.Status, operation.Issues); - - if (status.IsNotSuccess) - { - throw new YdbException(status); - } - - foreach (var entry in operation.Result.Unpack().Children) + string? tableType = null, + CancellationToken cancellationToken = default + ) => from ydbObject in await SchemaObjects(ydbConnection, cancellationToken) + let type = ydbObject.IsSystem + ? "SYSTEM_TABLE" + : ydbObject.Type switch { - var tablePath = fullPath[databasePath.Length..] + entry.Name; - - switch (entry.Type) - { - case Entry.Types.Type.Table: - var type = tablePath.IsSystem() ? "SYSTEM_TABLE" : "TABLE"; - if (type.IsPattern(tableType)) - { - tables.Add((tablePath, type)); - } - - break; - case Entry.Types.Type.ColumnTable: - if ("COLUMN_TABLE".IsPattern(tableType)) - { - tables.Add((tablePath, "COLUMN_TABLE")); - } - - break; - case Entry.Types.Type.Directory: - tables.AddRange( - await ListTables(ydbConnection, databasePath, fullPath + entry.Name, tableType, - cancellationToken) - ); - break; - case Entry.Types.Type.Unspecified: - case Entry.Types.Type.PersQueueGroup: - case Entry.Types.Type.Database: - case Entry.Types.Type.RtmrVolume: - case Entry.Types.Type.BlockStoreVolume: - case Entry.Types.Type.CoordinationNode: - case Entry.Types.Type.ColumnStore: - case Entry.Types.Type.Sequence: - case Entry.Types.Type.Replication: - case Entry.Types.Type.Topic: - default: - continue; - } + SchemeType.Table => "TABLE", + SchemeType.ColumnTable => "COLUMN_TABLE", + SchemeType.ExternalTable => "EXTERNAL_TABLE", + _ => null } - - return tables; - } - catch (Driver.TransportException e) - { - throw new YdbException("Transport error on ListDirectory", e); - } - } + where type != null && type.IsPattern(tableType) + select (ydbObject.Name, type); private static async Task GetDataSourceInformation(YdbConnection ydbConnection) { @@ -444,11 +389,83 @@ private static DataTable GetRestrictions() return table; } - private static string WithSuffix(string path) => path.EndsWith('/') ? path : path + '/'; + private static async Task> SchemaObjects( + YdbConnection ydbConnection, + string databasePath, + string path, + CancellationToken cancellationToken + ) + { + try + { + var fullPath = WithSuffix(path); + var ydbSchemaObjects = new List(); + var response = await ydbConnection.Session.Driver.UnaryCall( + SchemeService.ListDirectoryMethod, + new ListDirectoryRequest { Path = fullPath }, + new GrpcRequestSettings + { + TransportTimeout = TimeSpan.FromSeconds(TransportTimeoutSeconds), + CancellationToken = cancellationToken + } + ); + + var operation = response.Operation; + var status = Status.FromProto(operation.Status, operation.Issues); + + if (status.IsNotSuccess) + { + throw new YdbException(status); + } + + foreach (var entry in operation.Result.Unpack().Children) + { + var ydbObjectPath = fullPath[databasePath.Length..] + entry.Name; + + + switch (entry.Type) + { + case Entry.Types.Type.Directory: + ydbSchemaObjects.AddRange( + await SchemaObjects( + ydbConnection, + databasePath, + fullPath + entry.Name, + cancellationToken + ) + ); + break; + case Entry.Types.Type.Table: + case Entry.Types.Type.ColumnTable: + case Entry.Types.Type.Unspecified: + case Entry.Types.Type.PersQueueGroup: + case Entry.Types.Type.Database: + case Entry.Types.Type.RtmrVolume: + case Entry.Types.Type.BlockStoreVolume: + case Entry.Types.Type.CoordinationNode: + case Entry.Types.Type.ColumnStore: + case Entry.Types.Type.Sequence: + case Entry.Types.Type.Replication: + case Entry.Types.Type.Topic: + case Entry.Types.Type.ExternalTable: + case Entry.Types.Type.ExternalDataSource: + case Entry.Types.Type.View: + ydbSchemaObjects.Add(new YdbObject(entry.Type, ydbObjectPath)); + break; + default: + continue; + } + } - private static bool IsSystem(this string tablePath) => tablePath.StartsWith(".sys/") - || tablePath.StartsWith(".sys_health/") - || tablePath.StartsWith(".sys_health_dev/"); + return ydbSchemaObjects; + } + catch (Driver.TransportException e) + { + throw new YdbException("Transport error on ListDirectory", e); + } + } + + private static string WithSuffix(string path) => path.EndsWith('/') ? path : path + '/'; private static bool IsPattern(this string tableType, string? expectedTableType) => expectedTableType == null || expectedTableType.Equals(tableType, StringComparison.OrdinalIgnoreCase); diff --git a/src/Ydb.Sdk/src/Properties/AssemblyInfo.cs b/src/Ydb.Sdk/src/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..065a401c --- /dev/null +++ b/src/Ydb.Sdk/src/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("EfCore.Ydb")] +[assembly: InternalsVisibleTo("Ydb.Sdk.Tests")] diff --git a/src/Ydb.Sdk/src/Ydb.Sdk.csproj b/src/Ydb.Sdk/src/Ydb.Sdk.csproj index 28f07540..3f3462d9 100644 --- a/src/Ydb.Sdk/src/Ydb.Sdk.csproj +++ b/src/Ydb.Sdk/src/Ydb.Sdk.csproj @@ -36,10 +36,4 @@ - - - - <_Parameter1>Ydb.Sdk.Tests - - diff --git a/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs b/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs index c8249c11..b09e32e4 100644 --- a/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs +++ b/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs @@ -6,6 +6,8 @@ namespace Ydb.Sdk.Tests.Ado; +[CollectionDefinition("YdbSchemaTests isolation test", DisableParallelization = true)] +[Collection("YdbSchemaTests isolation test")] public class YdbSchemaTests : YdbAdoNetFixture { private readonly string _table1; @@ -45,9 +47,9 @@ public async Task GetSchema_WhenTablesCollection_ReturnAllTables() Assert.Equal(_table2, singleTable2.Rows[0]["table_name"].ToString()); Assert.Equal("TABLE", singleTable2.Rows[0]["table_type"].ToString()); - // not found case - var notFound = await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }); - Assert.Equal(0, notFound.Rows.Count); + await Assert.ThrowsAsync( + async () => await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }) + ); } [Fact] @@ -85,8 +87,9 @@ public async Task GetSchema_WhenTablesWithStatsCollection_ReturnAllTables() Assert.NotNull(singleTable2.Rows[0]["modification_time"]); // not found case - var notFound = await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }); - Assert.Equal(0, notFound.Rows.Count); + await Assert.ThrowsAsync( + async () => await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }) + ); } [Fact]