Skip to content

Commit a388213

Browse files
jadewang-dbjadewang-db
committed
rebase to master
1 parent 56b7aa2 commit a388213

15 files changed

+61
-78
lines changed

‎csharp/src/Drivers/Apache/Hive2/HiveServer2Statement.cs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,13 @@
2020
using System.Linq;
2121
using System.Threading;
2222
using System.Threading.Tasks;
23-
using Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch;
2423
using Apache.Arrow.Ipc;
2524
using Apache.Hive.Service.Rpc.Thrift;
2625
using Thrift.Transport;
2726

2827
namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
2928
{
30-
internal class HiveServer2Statement : AdbcStatement, IHiveServer2Statement
29+
internal class HiveServer2Statement : AdbcStatement
3130
{
3231
private const string GetPrimaryKeysCommandName = "getprimarykeys";
3332
private const string GetCrossReferenceCommandName = "getcrossreference";
@@ -284,9 +283,6 @@ protected internal int QueryTimeoutSeconds
284283

285284
public TOperationHandle? OperationHandle { get; private set; }
286285

287-
// Cast the Client to IAsync for CloudFetch compatibility
288-
TCLIService.IAsync IHiveServer2Statement.Client => Connection.Client;
289-
290286
// Keep the original Client property for internal use
291287
public TCLIService.Client Client => Connection.Client;
292288

‎csharp/src/Drivers/Apache/Spark/SparkParameters.cs‎

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -33,49 +33,6 @@ public class SparkParameters
3333
public const string Type = "adbc.spark.type";
3434
public const string DataTypeConv = "adbc.spark.data_type_conv";
3535
public const string ConnectTimeoutMilliseconds = "adbc.spark.connect_timeout_ms";
36-
37-
// CloudFetch configuration parameters
38-
/// <summary>
39-
/// Maximum number of retry attempts for CloudFetch downloads.
40-
/// Default value is 3 if not specified.
41-
/// </summary>
42-
public const string CloudFetchMaxRetries = "adbc.spark.cloudfetch.max_retries";
43-
44-
/// <summary>
45-
/// Delay in milliseconds between CloudFetch retry attempts.
46-
/// Default value is 500ms if not specified.
47-
/// </summary>
48-
public const string CloudFetchRetryDelayMs = "adbc.spark.cloudfetch.retry_delay_ms";
49-
50-
/// <summary>
51-
/// Timeout in minutes for CloudFetch HTTP operations.
52-
/// Default value is 5 minutes if not specified.
53-
/// </summary>
54-
public const string CloudFetchTimeoutMinutes = "adbc.spark.cloudfetch.timeout_minutes";
55-
56-
/// <summary>
57-
/// Maximum number of parallel downloads for CloudFetch operations.
58-
/// Default value is 3 if not specified.
59-
/// </summary>
60-
public const string CloudFetchParallelDownloads = "adbc.spark.cloudfetch.parallel_downloads";
61-
62-
/// <summary>
63-
/// Number of files to prefetch in CloudFetch operations.
64-
/// Default value is 2 if not specified.
65-
/// </summary>
66-
public const string CloudFetchPrefetchCount = "adbc.spark.cloudfetch.prefetch_count";
67-
68-
/// <summary>
69-
/// Maximum memory buffer size in MB for CloudFetch prefetched files.
70-
/// Default value is 200MB if not specified.
71-
/// </summary>
72-
public const string CloudFetchMemoryBufferSize = "adbc.spark.cloudfetch.memory_buffer_size_mb";
73-
74-
/// <summary>
75-
/// Whether CloudFetch prefetch functionality is enabled.
76-
/// Default value is true if not specified.
77-
/// </summary>
78-
public const string CloudFetchPrefetchEnabled = "adbc.spark.cloudfetch.prefetch_enabled";
7936
}
8037

8138
public static class SparkAuthTypeConstants

‎csharp/src/Drivers/Databricks/CloudFetch/CloudFetchDownloadManager.cs‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@
2222
using System.Threading;
2323
using System.Threading.Tasks;
2424
using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
25+
using Apache.Arrow.Adbc.Drivers.Databricks;
2526

26-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
27+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2728
{
2829
/// <summary>
2930
/// Manages the CloudFetch download pipeline.
@@ -36,7 +37,7 @@ internal sealed class CloudFetchDownloadManager : ICloudFetchDownloadManager
3637
private const int DefaultMemoryBufferSizeMB = 200;
3738
private const bool DefaultPrefetchEnabled = true;
3839

39-
private readonly HiveServer2Statement _statement;
40+
private readonly DatabricksStatement _statement;
4041
private readonly Schema _schema;
4142
private readonly bool _isLz4Compressed;
4243
private readonly ICloudFetchMemoryBufferManager _memoryManager;
@@ -55,7 +56,7 @@ internal sealed class CloudFetchDownloadManager : ICloudFetchDownloadManager
5556
/// <param name="statement">The HiveServer2 statement.</param>
5657
/// <param name="schema">The Arrow schema.</param>
5758
/// <param name="isLz4Compressed">Whether the results are LZ4 compressed.</param>
58-
public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema, bool isLz4Compressed)
59+
public CloudFetchDownloadManager(DatabricksStatement statement, Schema schema, bool isLz4Compressed)
5960
{
6061
_statement = statement ?? throw new ArgumentNullException(nameof(statement));
6162
_schema = schema ?? throw new ArgumentNullException(nameof(schema));
@@ -66,7 +67,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
6667

6768
// Parse parallel downloads
6869
int parallelDownloads = DefaultParallelDownloads;
69-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchParallelDownloads, out string? parallelDownloadsStr) &&
70+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchParallelDownloads, out string? parallelDownloadsStr) &&
7071
int.TryParse(parallelDownloadsStr, out int parsedParallelDownloads) &&
7172
parsedParallelDownloads > 0)
7273
{
@@ -75,7 +76,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
7576

7677
// Parse prefetch count
7778
int prefetchCount = DefaultPrefetchCount;
78-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchPrefetchCount, out string? prefetchCountStr) &&
79+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchPrefetchCount, out string? prefetchCountStr) &&
7980
int.TryParse(prefetchCountStr, out int parsedPrefetchCount) &&
8081
parsedPrefetchCount > 0)
8182
{
@@ -84,7 +85,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
8485

8586
// Parse memory buffer size
8687
int memoryBufferSizeMB = DefaultMemoryBufferSizeMB;
87-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchMemoryBufferSize, out string? memoryBufferSizeStr) &&
88+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchMemoryBufferSize, out string? memoryBufferSizeStr) &&
8889
int.TryParse(memoryBufferSizeStr, out int parsedMemoryBufferSize) &&
8990
parsedMemoryBufferSize > 0)
9091
{
@@ -93,7 +94,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
9394

9495
// Parse max retries
9596
int maxRetries = 3;
96-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchMaxRetries, out string? maxRetriesStr) &&
97+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchMaxRetries, out string? maxRetriesStr) &&
9798
int.TryParse(maxRetriesStr, out int parsedMaxRetries) &&
9899
parsedMaxRetries > 0)
99100
{
@@ -102,7 +103,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
102103

103104
// Parse retry delay
104105
int retryDelayMs = 500;
105-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchRetryDelayMs, out string? retryDelayStr) &&
106+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchRetryDelayMs, out string? retryDelayStr) &&
106107
int.TryParse(retryDelayStr, out int parsedRetryDelay) &&
107108
parsedRetryDelay > 0)
108109
{
@@ -111,7 +112,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
111112

112113
// Parse timeout minutes
113114
int timeoutMinutes = 5;
114-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchTimeoutMinutes, out string? timeoutStr) &&
115+
if (connectionProps.TryGetValue(DatabricksParameters.CloudFetchTimeoutMinutes, out string? timeoutStr) &&
115116
int.TryParse(timeoutStr, out int parsedTimeout) &&
116117
parsedTimeout > 0)
117118
{
@@ -160,7 +161,7 @@ public CloudFetchDownloadManager(HiveServer2Statement statement, Schema schema,
160161
/// <param name="resultFetcher">The result fetcher.</param>
161162
/// <param name="downloader">The downloader.</param>
162163
internal CloudFetchDownloadManager(
163-
HiveServer2Statement statement,
164+
DatabricksStatement statement,
164165
Schema schema,
165166
bool isLz4Compressed,
166167
ICloudFetchResultFetcher resultFetcher,

‎csharp/src/Drivers/Databricks/CloudFetch/CloudFetchDownloader.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
using System.Threading.Tasks;
2525
using K4os.Compression.LZ4.Streams;
2626

27-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
27+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2828
{
2929
/// <summary>
3030
/// Downloads files from URLs.

‎csharp/src/Drivers/Databricks/CloudFetch/CloudFetchMemoryBufferManager.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
using System.Threading;
2020
using System.Threading.Tasks;
2121

22-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
22+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2323
{
2424
/// <summary>
2525
/// Manages memory allocation for prefetched files.
@@ -132,4 +132,4 @@ public void ReleaseMemory(long size)
132132
}
133133
}
134134
}
135-
}
135+
}

‎csharp/src/Drivers/Databricks/CloudFetch/CloudFetchReader.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
using System.Net.Http;
2222
using System.Threading;
2323
using System.Threading.Tasks;
24-
using Apache.Arrow.Adbc.Drivers.Apache;
24+
using Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch;
25+
using Apache.Arrow.Adbc.Drivers.Databricks;
2526
using Apache.Arrow.Ipc;
2627
using Apache.Hive.Service.Rpc.Thrift;
2728

@@ -55,7 +56,7 @@ public CloudFetchReader(DatabricksStatement statement, Schema schema, bool isLz4
5556
// Check if prefetch is enabled
5657
var connectionProps = statement.Connection.Properties;
5758
isPrefetchEnabled = true; // Default to true
58-
if (connectionProps.TryGetValue(SparkParameters.CloudFetchPrefetchEnabled, out string? prefetchEnabledStr) &&
59+
if (connectionProps.TryGetValue(Databricks.DatabricksParameters.CloudFetchPrefetchEnabled, out string? prefetchEnabledStr) &&
5960
bool.TryParse(prefetchEnabledStr, out bool parsedPrefetchEnabled))
6061
{
6162
isPrefetchEnabled = parsedPrefetchEnabled;
@@ -191,7 +192,7 @@ private void ThrowIfDisposed()
191192
{
192193
if (isDisposed)
193194
{
194-
throw new ObjectDisposedException(nameof(SparkCloudFetchReader));
195+
throw new ObjectDisposedException(nameof(CloudFetchReader));
195196
}
196197
}
197198
}

‎csharp/src/Drivers/Databricks/CloudFetch/CloudFetchResultFetcher.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
using System.Threading.Tasks;
2424
using Apache.Hive.Service.Rpc.Thrift;
2525

26-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
26+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2727
{
2828
/// <summary>
2929
/// Fetches result chunks from the Thrift server.
@@ -229,4 +229,4 @@ private async Task FetchNextResultBatchAsync(CancellationToken cancellationToken
229229
}
230230
}
231231
}
232-
}
232+
}

‎csharp/src/Drivers/Databricks/CloudFetch/DownloadResult.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
using System.Threading.Tasks;
2121
using Apache.Hive.Service.Rpc.Thrift;
2222

23-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
23+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2424
{
2525
/// <summary>
2626
/// Represents a downloaded result file with its associated metadata.

‎csharp/src/Drivers/Databricks/CloudFetch/EndOfResultsGuard.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
using System.Threading.Tasks;
2121
using Apache.Hive.Service.Rpc.Thrift;
2222

23-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
23+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2424
{
2525
/// <summary>
2626
/// Special marker class that indicates the end of results in the download queue.
@@ -66,4 +66,4 @@ public void Dispose()
6666
// Nothing to dispose
6767
}
6868
}
69-
}
69+
}

‎csharp/src/Drivers/Databricks/CloudFetch/ICloudFetchInterfaces.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
using System.Threading.Tasks;
2323
using Apache.Hive.Service.Rpc.Thrift;
2424

25-
namespace Apache.Arrow.Adbc.Drivers.Apache.Spark.CloudFetch
25+
namespace Apache.Arrow.Adbc.Drivers.Apache.Databricks.CloudFetch
2626
{
2727
/// <summary>
2828
/// Represents a downloaded result file with its associated metadata.

0 commit comments

Comments
 (0)