feat: added support columns collection (#262)

KirillKurdyukov · web-flow · commit ae784b6e80cc · 2025-02-02T19:14:26.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,6 @@
+* Fixed: YdbDataReader.GetDataTypeName for optional values.
+* Added support for "Columns" collectionName in YdbConnection.GetSchema(Async).
+
 ## v0.12.0
 * GetUint64(int ordinal) returns a ulong for Uint8, Uint16, Uint32, Uint64 YDB types.
 * GetInt64(int ordinal) returns a int for Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32 YDB types.
diff --git a/src/Ydb.Sdk/src/Ado/YdbDataReader.cs b/src/Ydb.Sdk/src/Ado/YdbDataReader.cs
@@ -181,7 +181,7 @@ private static void CheckOffsets<T>(long dataOffset, T[]? buffer, int bufferOffs
 
     public override string GetDataTypeName(int ordinal)
     {
-        return ReaderMetadata.GetColumn(ordinal).Type.TypeId.ToString();
+        return ReaderMetadata.GetColumn(ordinal).Type.YqlTableType();
     }
 
     public override DateTime GetDateTime(int ordinal)
diff --git a/src/Ydb.Sdk/src/Ado/YdbSchema.cs b/src/Ydb.Sdk/src/Ado/YdbSchema.cs
@@ -1,5 +1,7 @@
+using System.Collections.Immutable;
 using System.Data;
 using System.Data.Common;
+using System.Globalization;
 using Ydb.Scheme;
 using Ydb.Scheme.V1;
 using Ydb.Sdk.Services.Table;
@@ -28,6 +30,7 @@ public static Task<DataTable> GetSchemaAsync(
 
             // Ydb specific Schema Collections
             "TABLES" => GetTables(ydbConnection, restrictions, cancellationToken),
+            "COLUMNS" => GetColumns(ydbConnection, restrictions, cancellationToken),
             "TABLESWITHSTATS" => GetTablesWithStats(ydbConnection, restrictions, cancellationToken),
 
             _ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName,
@@ -40,18 +43,24 @@ private static async Task<DataTable> GetTables(
         string?[] restrictions,
         CancellationToken cancellationToken)
     {
-        var table = new DataTable("Tables");
-        table.Columns.Add("table_name", typeof(string));
-        table.Columns.Add("table_type", typeof(string));
+        var table = new DataTable("Tables")
+        {
+            Locale = CultureInfo.InvariantCulture,
+            Columns =
+            {
+                new DataColumn("table_name"),
+                new DataColumn("table_type")
+            }
+        };
 
         var tableName = restrictions[0];
         var tableType = restrictions[1];
         var database = ydbConnection.Database;
 
         if (tableName == null) // tableName isn't set
         {
-            foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver,
-                         WithSuffix(database), database, tableType, cancellationToken))
+            foreach (var tupleTable in
+                     await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken))
             {
                 table.Rows.Add(tupleTable.TableName, tupleTable.TableType);
             }
@@ -61,7 +70,6 @@ private static async Task<DataTable> GetTables(
             await AppendDescribeTable(
                 ydbConnection: ydbConnection,
                 describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken },
-                database: database,
                 tableName: tableName,
                 tableType: tableType,
                 (_, type) => { table.Rows.Add(tableName, type); });
@@ -75,27 +83,32 @@ private static async Task<DataTable> GetTablesWithStats(
         string?[] restrictions,
         CancellationToken cancellationToken)
     {
-        var table = new DataTable("TablesWithStats");
-        table.Columns.Add("table_name", typeof(string));
-        table.Columns.Add("table_type", typeof(string));
-        table.Columns.Add("rows_estimate", typeof(ulong));
-        table.Columns.Add("creation_time", typeof(DateTime));
-        table.Columns.Add("modification_time", typeof(DateTime));
+        var table = new DataTable("TablesWithStats")
+        {
+            Locale = CultureInfo.InvariantCulture,
+            Columns =
+            {
+                new DataColumn("table_name"),
+                new DataColumn("table_type"),
+                new DataColumn("rows_estimate", typeof(ulong)),
+                new DataColumn("creation_time", typeof(DateTime)),
+                new DataColumn("modification_time", typeof(DateTime))
+            }
+        };
 
         var tableName = restrictions[0];
         var tableType = restrictions[1];
         var database = ydbConnection.Database;
 
         if (tableName == null) // tableName isn't set
         {
-            foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver,
-                         WithSuffix(database), database, tableType, cancellationToken))
+            foreach (var tupleTable in
+                     await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken))
             {
                 await AppendDescribeTable(
                     ydbConnection: ydbConnection,
                     describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken }
                         .WithTableStats(),
-                    database: database,
                     tableName: tupleTable.TableName,
                     tableType: tableType,
                     (describeTableResult, type) =>
@@ -117,7 +130,6 @@ await AppendDescribeTable(
                 ydbConnection: ydbConnection,
                 describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken }
                     .WithTableStats(),
-                database: database,
                 tableName: tableName,
                 tableType: tableType,
                 (describeTableResult, type) =>
@@ -136,18 +148,74 @@ await AppendDescribeTable(
         return table;
     }
 
+    private static async Task<DataTable> GetColumns(
+        YdbConnection ydbConnection,
+        string?[] restrictions,
+        CancellationToken cancellationToken)
+    {
+        var table = new DataTable("Columns")
+        {
+            Locale = CultureInfo.InvariantCulture,
+            Columns =
+            {
+                new DataColumn("table_name"),
+                new DataColumn("column_name"),
+                new DataColumn("ordinal_position", typeof(int)),
+                new DataColumn("is_nullable"),
+                new DataColumn("data_type"),
+                new DataColumn("family_name")
+            }
+        };
+        var tableNameRestriction = restrictions[0];
+        var columnName = restrictions[1];
+
+        var tableNames = await ListTableNames(ydbConnection, tableNameRestriction, cancellationToken);
+        foreach (var tableName in tableNames)
+        {
+            await AppendDescribeTable(
+                ydbConnection,
+                new DescribeTableSettings { CancellationToken = cancellationToken },
+                tableName,
+                null,
+                (result, _) =>
+                {
+                    for (var ordinal = 0; ordinal < result.Columns.Count; ordinal++)
+                    {
+                        var column = result.Columns[ordinal];
+
+                        if (!column.Name.IsPattern(columnName))
+                        {
+                            continue;
+                        }
+
+                        var row = table.Rows.Add();
+                        var type = column.Type;
+
+                        row["table_name"] = tableName;
+                        row["column_name"] = column.Name;
+                        row["ordinal_position"] = ordinal;
+                        row["is_nullable"] = type.TypeCase == Type.TypeOneofCase.OptionalType ? "YES" : "NO";
+                        row["data_type"] = type.YqlTableType();
+                        row["family_name"] = column.Family;
+                    }
+                }
+            );
+        }
+
+        return table;
+    }
+
     private static async Task AppendDescribeTable(
         YdbConnection ydbConnection,
         DescribeTableSettings describeTableSettings,
-        string database,
         string tableName,
         string? tableType,
         Action<DescribeTableResult, string> appendInTable)
     {
         try
         {
             var describeResponse = await ydbConnection.Session
-                .DescribeTable(WithSuffix(database) + tableName, describeTableSettings);
+                .DescribeTable(WithSuffix(ydbConnection.Database) + tableName, describeTableSettings);
 
             if (describeResponse.Operation.Status == StatusIds.Types.StatusCode.SchemeError)
             {
@@ -174,7 +242,7 @@ private static async Task AppendDescribeTable(
                 _ => throw new YdbException($"Unexpected entry type for Table: {describeRes.Self.Type}")
             };
 
-            if (type.IsTableType(tableType))
+            if (type.IsPattern(tableType))
             {
                 appendInTable(describeRes, type);
             }
@@ -187,8 +255,26 @@ private static async Task AppendDescribeTable(
         }
     }
 
-    private static async Task<List<(string TableName, string TableType)>> ListTables(
-        Driver driver,
+    private static async Task<IReadOnlyCollection<string>> ListTableNames(
+        YdbConnection ydbConnection,
+        string? tableName,
+        CancellationToken cancellationToken)
+    {
+        var database = ydbConnection.Database;
+
+        return tableName != null
+            ? new List<string> { tableName }
+            : (await ListTables(
+                ydbConnection,
+                WithSuffix(database),
+                database,
+                null,
+                cancellationToken
+            )).Select(tuple => tuple.TableName).ToImmutableList();
+    }
+
+    private static async Task<IReadOnlyCollection<(string TableName, string TableType)>> ListTables(
+        YdbConnection ydbConnection,
         string databasePath,
         string path,
         string? tableType,
@@ -198,7 +284,7 @@ private static async Task AppendDescribeTable(
         {
             var fullPath = WithSuffix(path);
             var tables = new List<(string, string)>();
-            var response = await driver.UnaryCall(
+            var response = await ydbConnection.Session.Driver.UnaryCall(
                 SchemeService.ListDirectoryMethod,
                 new ListDirectoryRequest { Path = fullPath },
                 new GrpcRequestSettings { CancellationToken = cancellationToken }
@@ -220,22 +306,23 @@ private static async Task AppendDescribeTable(
                 {
                     case Entry.Types.Type.Table:
                         var type = tablePath.IsSystem() ? "SYSTEM_TABLE" : "TABLE";
-                        if (type.IsTableType(tableType))
+                        if (type.IsPattern(tableType))
                         {
                             tables.Add((tablePath, type));
                         }
 
                         break;
                     case Entry.Types.Type.ColumnTable:
-                        if ("COLUMN_TABLE".IsTableType(tableType))
+                        if ("COLUMN_TABLE".IsPattern(tableType))
                         {
                             tables.Add((tablePath, "COLUMN_TABLE"));
                         }
 
                         break;
                     case Entry.Types.Type.Directory:
                         tables.AddRange(
-                            await ListTables(driver, databasePath, fullPath + entry.Name, tableType, cancellationToken)
+                            await ListTables(ydbConnection, databasePath, fullPath + entry.Name, tableType,
+                                cancellationToken)
                         );
                         break;
                     case Entry.Types.Type.Unspecified:
@@ -333,6 +420,7 @@ private static DataTable GetMetaDataCollections()
         // Ydb Specific Schema Collections
         table.Rows.Add("Tables", 2, 1);
         table.Rows.Add("TablesWithStats", 2, 1);
+        table.Rows.Add("Columns", 2, 2);
 
         return table;
     }
@@ -350,6 +438,8 @@ private static DataTable GetRestrictions()
         table.Rows.Add("Tables", "TableType", "TABLE_TYPE", 2);
         table.Rows.Add("TablesWithStats", "Table", "TABLE_NAME", 1);
         table.Rows.Add("TablesWithStats", "TableType", "TABLE_TYPE", 2);
+        table.Rows.Add("Columns", "Table", "TABLE_NAME", 1);
+        table.Rows.Add("Columns", "Column", "COLUMN_NAME", 2);
 
         return table;
     }
@@ -366,8 +456,15 @@ private static bool IsSystem(this string tablePath)
                || tablePath.StartsWith(".sys_health_dev/");
     }
 
-    private static bool IsTableType(this string tableType, string? expectedTableType)
+    private static bool IsPattern(this string tableType, string? expectedTableType)
     {
         return expectedTableType == null || expectedTableType.Equals(tableType, StringComparison.OrdinalIgnoreCase);
     }
+
+    internal static string YqlTableType(this Type type)
+    {
+        return type.TypeCase == Type.TypeOneofCase.OptionalType
+            ? type.OptionalType.Item.TypeId.ToString()
+            : type.TypeId.ToString();
+    }
 }
diff --git a/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs b/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs
@@ -196,6 +196,9 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade
         Assert.True(ydbDataReader.HasRows);
         // Read 2 result set
         Assert.True(await ydbDataReader.NextResultAsync());
+        Assert.Equal("Bool", ydbDataReader.GetDataTypeName(0));
+        Assert.Equal("Double", ydbDataReader.GetDataTypeName(1));
+        Assert.Equal("Int32", ydbDataReader.GetDataTypeName(2));
         for (var i = 0; i < 1500; i++)
         {
             // Read meta info 
@@ -214,12 +217,19 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade
 
         // Read 3 result set
         Assert.True(await ydbDataReader.NextResultAsync());
+        Assert.Equal("Int8", ydbDataReader.GetDataTypeName(0));
+        Assert.Equal("null_field", ydbDataReader.GetName(0));
         Assert.True(await ydbDataReader.ReadAsync());
         Assert.True(ydbDataReader.IsDBNull(0));
+        Assert.Equal(DBNull.Value, ydbDataReader.GetValue(0));
         Assert.False(await ydbDataReader.ReadAsync());
 
         // Read 4 result set
         Assert.True(await ydbDataReader.NextResultAsync());
+        Assert.Equal("Datetime", ydbDataReader.GetDataTypeName(0));
+        Assert.Equal("Key", ydbDataReader.GetName(0));
+        Assert.Equal("Timestamp", ydbDataReader.GetDataTypeName(1));
+        Assert.Equal("Value", ydbDataReader.GetName(1));
         Assert.True(await ydbDataReader.ReadAsync());
         Assert.Equal(dateTime, ydbDataReader.GetDateTime(0));
         Assert.Equal(timestamp, ydbDataReader.GetDateTime(1));
diff --git a/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs b/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs

Original file line number	Diff line number	Diff line change
`@@ -181,7 +181,7 @@ private static void CheckOffsets<T>(long dataOffset, T[]? buffer, int bufferOffs`
`181`	`181`
`182`	`182`	`public override string GetDataTypeName(int ordinal)`
`183`	`183`	`{`
`184`		`- return ReaderMetadata.GetColumn(ordinal).Type.TypeId.ToString();`
	`184`	`+ return ReaderMetadata.GetColumn(ordinal).Type.YqlTableType();`
`185`	`185`	`}`
`186`	`186`
`187`	`187`	`public override DateTime GetDateTime(int ordinal)`