Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
b850825
feat(csharp): implement metadata operations for Statement Execution API
msrathore-db Dec 23, 2025
2c4d461
feat(csharp): add E2E tests for Statement Execution API metadata
msrathore-db Dec 23, 2025
696fde6
fix(csharp): align E2E tests with existing test framework
msrathore-db Dec 23, 2025
89b8bde
docs: add comprehensive task status for metadata implementation
msrathore-db Dec 23, 2025
af096f4
docs(csharp): add comprehensive XML documentation for metadata methods
msrathore-db Dec 23, 2025
ab5d888
test(csharp): add comprehensive unit tests for metadata helper methods
msrathore-db Dec 23, 2025
c23c6ef
docs(csharp): add comprehensive metadata operations section to README
msrathore-db Dec 23, 2025
4ebf7da
docs: update task status to reflect completed quick wins
msrathore-db Dec 23, 2025
ace455e
feat(csharp): implement full nested structure for GetObjects(All)
msrathore-db Dec 23, 2025
24836ab
docs: update documentation to reflect completed GetObjects(All) imple…
msrathore-db Dec 23, 2025
738777d
docs: add comprehensive gap analysis for REST metadata vs Thrift/JDBC
msrathore-db Dec 23, 2025
9e82521
feat(csharp): implement GetInfo() for Statement Execution API
msrathore-db Dec 23, 2025
b97f13f
perf(csharp): add parallel execution for GetObjects metadata fetching
msrathore-db Dec 23, 2025
b884a74
docs: update task status to mark TASK_019 (parallel execution) complete
msrathore-db Dec 24, 2025
331d2ca
feat(csharp): add graceful error handling for permission denied scena…
msrathore-db Dec 24, 2025
3db5540
docs: update task status to mark TASK_023 (permission handling) complete
msrathore-db Dec 24, 2025
fde30c7
feat(csharp): implement metadata caching with configurable TTL
msrathore-db Dec 24, 2025
4e6a295
docs: update task status to mark caching tasks complete
msrathore-db Dec 24, 2025
192feae
docs: add metadata caching configuration section
msrathore-db Dec 24, 2025
3ee5430
feat(csharp): implement GetPrimaryKeys and GetImportedKeys for Statem…
msrathore-db Dec 24, 2025
288daa1
test(csharp): add E2E tests for GetPrimaryKeys and GetImportedKeys
msrathore-db Dec 24, 2025
2774a26
docs(csharp): update TASK_STATUS.md - GetPrimaryKeys and GetImportedK…
msrathore-db Dec 24, 2025
7b00518
Initial draft of complete implementation of metadata in SEA
msrathore-db Dec 29, 2025
d90f208
Refactored databricksStatement and SEA connection to use a common met…
msrathore-db Dec 29, 2025
36fe05c
Improved logging in the SEA metadata
msrathore-db Dec 29, 2025
89565c9
Completed the code for metadata related methods
msrathore-db Jan 1, 2026
aabe321
fix(csharp): resolve linting errors and remove docs from tracking
msrathore-db Jan 1, 2026
271acfa
Merge branch 'main' into feature/sea-metadata-implementation
msrathore-db Jan 1, 2026
67730b0
refactor(csharp): simplify tests and reorganize documentation
msrathore-db Jan 1, 2026
658c716
fix(csharp): add missing newlines at end of files
msrathore-db Jan 1, 2026
6ceca01
Added text file to show the output of the metadata calls between SEA …
msrathore-db Jan 1, 2026
d7678ef
Fixed tests
msrathore-db Jan 1, 2026
96fc1eb
Fixed the failing tests
msrathore-db Jan 1, 2026
3c30756
Fixed the data types and modified the design doc
msrathore-db Jan 2, 2026
e8ab35d
Reverted changes in the code without metadata
msrathore-db Jan 2, 2026
ae106df
fix(ci): add repository checkout step before PR title validation
msrathore-db Jan 2, 2026
c264e95
Added tracing to StatementConnection and also added the param of esca…
msrathore-db Jan 7, 2026
88bfb75
Merge branch 'main' into feature/sea-metadata-implementation
msrathore-db Jan 8, 2026
705e7b2
Corrected tests to remove warnings
msrathore-db Jan 8, 2026
6d1b4fc
Updated the code to fallback to the connection catalog in case of nul…
msrathore-db Jan 9, 2026
14105b9
Null checkers added
msrathore-db Jan 11, 2026
3438a9c
Added some more unit tests for databricksTypeMapper and metadatautili…
msrathore-db Jan 12, 2026
36de4d5
Added test for SQLCommandBuilder
msrathore-db Jan 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
483 changes: 174 additions & 309 deletions csharp/doc/statement-execution-api-design.md

Large diffs are not rendered by default.

3,192 changes: 3,192 additions & 0 deletions csharp/examples/metadata_comparison.txt

Large diffs are not rendered by default.

310 changes: 310 additions & 0 deletions csharp/src/ColumnMetadataSchemas.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,310 @@
/*
* Copyright (c) 2025 ADBC Drivers Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using Apache.Arrow;
using Apache.Arrow.Types;

namespace AdbcDrivers.Databricks
{
/// <summary>
/// Shared helper class for standard metadata schemas.
/// Used by both Thrift and Statement Execution API protocols to ensure consistent metadata structure.
/// Provides schemas for GetCatalogs, GetSchemas, GetTables, GetColumns, GetPrimaryKeys, GetCrossReference.
/// </summary>
internal static class ColumnMetadataSchemas
{
#region GetColumns Schema (24 columns)

/// <summary>
/// Creates the standard GetColumns schema.
/// This schema follows the standard flat table format for column metadata.
/// </summary>
/// <returns>Schema with 24 column definitions</returns>
public static Schema CreateColumnMetadataSchema()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

few question on this.

  1. for thrift base driver, why we don't need this?
  2. this definition to me looks like should be something in the ADBC base library.
  3. at least some shared code between thrift and SEA?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Arrow code
We've these definitions there as well.

{
var fields = new[]
{
new Field("TABLE_CAT", StringType.Default, true),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any difference of SEA vs Thrift metadata schema? I understand value can diff but schema should be same? In that case can we merge them? Maybe we need first refactor the adbc repo.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah that would need refactor in the ADBC repo. A lot of the code can be common in both. But I didn't make any changes in the arrow repo so had to implement the entire thing here. Should we refactor the adbc repo?

new Field("TABLE_SCHEM", StringType.Default, true),
new Field("TABLE_NAME", StringType.Default, true),
new Field("COLUMN_NAME", StringType.Default, true),
new Field("DATA_TYPE", Int32Type.Default, true),
new Field("TYPE_NAME", StringType.Default, true),
new Field("COLUMN_SIZE", Int32Type.Default, true),
new Field("BUFFER_LENGTH", Int8Type.Default, true),
new Field("DECIMAL_DIGITS", Int32Type.Default, true),
new Field("NUM_PREC_RADIX", Int32Type.Default, true),
new Field("NULLABLE", Int32Type.Default, true),
new Field("REMARKS", StringType.Default, true),
new Field("COLUMN_DEF", StringType.Default, true),
new Field("SQL_DATA_TYPE", Int32Type.Default, true),
new Field("SQL_DATETIME_SUB", Int32Type.Default, true),
new Field("CHAR_OCTET_LENGTH", Int32Type.Default, true),
new Field("ORDINAL_POSITION", Int32Type.Default, true),
new Field("IS_NULLABLE", StringType.Default, true),
new Field("SCOPE_CATALOG", StringType.Default, true),
new Field("SCOPE_SCHEMA", StringType.Default, true),
new Field("SCOPE_TABLE", StringType.Default, true),
new Field("SOURCE_DATA_TYPE", Int16Type.Default, true),
new Field("IS_AUTO_INCREMENT", StringType.Default, true),
new Field("BASE_TYPE_NAME", StringType.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for each column in the column metadata schema.
/// Useful for returning empty results from metadata queries.
/// </summary>
/// <returns>Array of empty Arrow arrays</returns>
public static IArrowArray[] CreateColumnMetadataEmptyArray()
{
return
[
new StringArray.Builder().Build(), // TABLE_CAT
new StringArray.Builder().Build(), // TABLE_SCHEM
new StringArray.Builder().Build(), // TABLE_NAME
new StringArray.Builder().Build(), // COLUMN_NAME
new Int32Array.Builder().Build(), // DATA_TYPE
new StringArray.Builder().Build(), // TYPE_NAME
new Int32Array.Builder().Build(), // COLUMN_SIZE
new Int8Array.Builder().Build(), // BUFFER_LENGTH
new Int32Array.Builder().Build(), // DECIMAL_DIGITS
new Int32Array.Builder().Build(), // NUM_PREC_RADIX
new Int32Array.Builder().Build(), // NULLABLE
new StringArray.Builder().Build(), // REMARKS
new StringArray.Builder().Build(), // COLUMN_DEF
new Int32Array.Builder().Build(), // SQL_DATA_TYPE
new Int32Array.Builder().Build(), // SQL_DATETIME_SUB
new Int32Array.Builder().Build(), // CHAR_OCTET_LENGTH
new Int32Array.Builder().Build(), // ORDINAL_POSITION
new StringArray.Builder().Build(), // IS_NULLABLE
new StringArray.Builder().Build(), // SCOPE_CATALOG
new StringArray.Builder().Build(), // SCOPE_SCHEMA
new StringArray.Builder().Build(), // SCOPE_TABLE
new Int16Array.Builder().Build(), // SOURCE_DATA_TYPE
new StringArray.Builder().Build(), // IS_AUTO_INCREMENT
new StringArray.Builder().Build() // BASE_TYPE_NAME
];
}

#endregion

#region GetCatalogs Schema (1 column)

/// <summary>
/// Creates the standard GetCatalogs schema (1 column: TABLE_CAT).
/// Follows JDBC DatabaseMetaData.getCatalogs() convention.
/// </summary>
/// <returns>Schema with TABLE_CAT column</returns>
public static Schema CreateCatalogsSchema()
{
var fields = new[]
{
new Field("TABLE_CAT", StringType.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for GetCatalogs schema.
/// </summary>
/// <returns>Array of empty Arrow arrays (1 column)</returns>
public static IArrowArray[] CreateCatalogsEmptyArray()
{
return
[
new StringArray.Builder().Build() // TABLE_CAT
];
}

#endregion

#region GetSchemas Schema (2 columns)

/// <summary>
/// Creates the standard GetSchemas schema (2 columns: TABLE_SCHEM, TABLE_CATALOG).
/// Follows JDBC DatabaseMetaData.getSchemas() convention.
/// </summary>
/// <returns>Schema with TABLE_SCHEM and TABLE_CATALOG columns</returns>
public static Schema CreateSchemasSchema()
{
var fields = new[]
{
new Field("TABLE_SCHEM", StringType.Default, true),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does the ordering matter here?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes the ordering is specified in the spec. The code adheres to it as well

new Field("TABLE_CATALOG", StringType.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for GetSchemas schema.
/// </summary>
/// <returns>Array of empty Arrow arrays (2 columns)</returns>
public static IArrowArray[] CreateSchemasEmptyArray()
{
return
[
new StringArray.Builder().Build(), // TABLE_SCHEM
new StringArray.Builder().Build() // TABLE_CATALOG
];
}

#endregion

#region GetTables Schema (10 columns)

/// <summary>
/// Creates the standard GetTables schema (10 columns).
/// Follows JDBC DatabaseMetaData.getTables() convention.
/// </summary>
/// <returns>Schema with 10 table metadata columns</returns>
public static Schema CreateTablesSchema()
{
var fields = new[]
{
new Field("TABLE_CAT", StringType.Default, true),
new Field("TABLE_SCHEM", StringType.Default, true),
new Field("TABLE_NAME", StringType.Default, true),
new Field("TABLE_TYPE", StringType.Default, true),
new Field("REMARKS", StringType.Default, true),
new Field("TYPE_CAT", StringType.Default, true),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does TYPE_CAT mean?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spec for getTables
Although this is always NULL in JDBC.

new Field("TYPE_SCHEM", StringType.Default, true),
new Field("TYPE_NAME", StringType.Default, true),
new Field("SELF_REFERENCING_COL_NAME", StringType.Default, true),
new Field("REF_GENERATION", StringType.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for GetTables schema.
/// </summary>
/// <returns>Array of empty Arrow arrays (10 columns)</returns>
public static IArrowArray[] CreateTablesEmptyArray()
{
return
[
new StringArray.Builder().Build(), // TABLE_CAT
new StringArray.Builder().Build(), // TABLE_SCHEM
new StringArray.Builder().Build(), // TABLE_NAME
new StringArray.Builder().Build(), // TABLE_TYPE
new StringArray.Builder().Build(), // REMARKS
new StringArray.Builder().Build(), // TYPE_CAT
new StringArray.Builder().Build(), // TYPE_SCHEM
new StringArray.Builder().Build(), // TYPE_NAME
new StringArray.Builder().Build(), // SELF_REFERENCING_COL_NAME
new StringArray.Builder().Build() // REF_GENERATION
];
}

#endregion

#region GetPrimaryKeys Schema (6 columns)

/// <summary>
/// Creates the standard GetPrimaryKeys schema (6 columns).
/// </summary>
/// <returns>Schema with 6 primary key metadata columns</returns>
public static Schema CreatePrimaryKeySchema()
{
var fields = new[]
{
new Field("TABLE_CAT", StringType.Default, true),
new Field("TABLE_SCHEM", StringType.Default, true),
new Field("TABLE_NAME", StringType.Default, true),
new Field("COLUMN_NAME", StringType.Default, true),
new Field("KEQ_SEQ", Int32Type.Default, true),
new Field("PK_NAME", StringType.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for GetPrimaryKeys schema.
/// </summary>
/// <returns>Array of empty Arrow arrays (6 columns)</returns>
public static IArrowArray[] CreatePrimaryKeyEmptyArray()
{
return
[
new StringArray.Builder().Build(), // TABLE_CAT
new StringArray.Builder().Build(), // TABLE_SCHEM
new StringArray.Builder().Build(), // TABLE_NAME
new StringArray.Builder().Build(), // COLUMN_NAME
new Int32Array.Builder().Build(), // KEQ_SEQ
new StringArray.Builder().Build() // PK_NAME
];
}

#endregion

#region GetCrossReference (Foreign Keys) Schema (14 columns)

/// <summary>
/// Creates the standard GetCrossReference schema (14 columns).
/// </summary>
/// <returns>Schema with 14 foreign key metadata columns</returns>
public static Schema CreateForeignKeySchema()
{
var fields = new[]
{
new Field("PKTABLE_CAT", StringType.Default, true),
new Field("PKTABLE_SCHEM", StringType.Default, true),
new Field("PKTABLE_NAME", StringType.Default, true),
new Field("PKCOLUMN_NAME", StringType.Default, true),
new Field("FKTABLE_CAT", StringType.Default, true),
new Field("FKTABLE_SCHEM", StringType.Default, true),
new Field("FKTABLE_NAME", StringType.Default, true),
new Field("FKCOLUMN_NAME", StringType.Default, true),
new Field("KEQ_SEQ", Int32Type.Default, true),
new Field("UPDATE_RULE", Int32Type.Default, true),
new Field("DELETE_RULE", Int32Type.Default, true),
new Field("FK_NAME", StringType.Default, true),
new Field("PK_NAME", StringType.Default, true),
new Field("DEFERRABILITY", Int32Type.Default, true)
};
return new Schema(fields, null);
}

/// <summary>
/// Creates empty Arrow arrays for GetCrossReference schema.
/// </summary>
/// <returns>Array of empty Arrow arrays (14 columns)</returns>
public static IArrowArray[] CreateForeignKeyEmptyArray()
{
return
[
new StringArray.Builder().Build(), // PKTABLE_CAT
new StringArray.Builder().Build(), // PKTABLE_SCHEM
new StringArray.Builder().Build(), // PKTABLE_NAME
new StringArray.Builder().Build(), // PKCOLUMN_NAME
new StringArray.Builder().Build(), // FKTABLE_CAT
new StringArray.Builder().Build(), // FKTABLE_SCHEM
new StringArray.Builder().Build(), // FKTABLE_NAME
new StringArray.Builder().Build(), // FKCOLUMN_NAME
new Int32Array.Builder().Build(), // KEQ_SEQ
new Int32Array.Builder().Build(), // UPDATE_RULE
new Int32Array.Builder().Build(), // DELETE_RULE
new StringArray.Builder().Build(), // FK_NAME
new StringArray.Builder().Build(), // PK_NAME
new Int32Array.Builder().Build() // DEFERRABILITY
];
}

#endregion
}
}
Loading
Loading