diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ConnectionOpenEvent.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ConnectionOpenEvent.cs new file mode 100644 index 0000000000..d9839f1b52 --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ConnectionOpenEvent.cs @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + /// + /// Tag definitions for Connection.Open events. + /// + internal static class ConnectionOpenEvent + { + public const string EventName = "Connection.Open"; + + // Identity + [TelemetryTag("workspace.id", ExportScope = TagExportScope.ExportAll, Required = true, Description = "Workspace ID")] + public const string WorkspaceId = "workspace.id"; + + [TelemetryTag("session.id", ExportScope = TagExportScope.ExportAll, Required = true, Description = "Session ID")] + public const string SessionId = "session.id"; + + // Driver Configuration + [TelemetryTag("driver.version", ExportScope = TagExportScope.ExportAll, Description = "Driver version")] + public const string DriverVersion = "driver.version"; + + [TelemetryTag("driver.os", ExportScope = TagExportScope.ExportAll, Description = "Operating system")] + public const string DriverOS = "driver.os"; + + [TelemetryTag("driver.runtime", ExportScope = TagExportScope.ExportAll, Description = ".NET runtime")] + public const string DriverRuntime = "driver.runtime"; + + // Feature Flags + [TelemetryTag("feature.cloudfetch", ExportScope = TagExportScope.ExportAll, Description = "CloudFetch enabled")] + public const string FeatureCloudFetch = "feature.cloudfetch"; + + [TelemetryTag("feature.lz4", ExportScope = TagExportScope.ExportAll, Description = "LZ4 compression enabled")] + public const string FeatureLz4 = "feature.lz4"; + + [TelemetryTag("feature.direct_results", ExportScope = TagExportScope.ExportAll, Description = "Direct results enabled")] + public const string FeatureDirectResults = "feature.direct_results"; + + [TelemetryTag("feature.multiple_catalog", ExportScope = TagExportScope.ExportAll, Description = "Multiple catalog enabled")] + public const string FeatureMultipleCatalog = "feature.multiple_catalog"; + + [TelemetryTag("feature.trace_propagation", ExportScope = TagExportScope.ExportAll, Description = "Trace propagation enabled")] + public const string FeatureTracePropagation = "feature.trace_propagation"; + + [TelemetryTag("server.address", ExportScope = TagExportScope.ExportLocal, Description = "Server address")] + public const string ServerAddress = "server.address"; + + /// + /// Returns tags allowed for Databricks export (privacy filter). + /// + public static IReadOnlyCollection GetDatabricksExportTags() + { + return new HashSet + { + WorkspaceId, + SessionId, + DriverVersion, + DriverOS, + DriverRuntime, + FeatureCloudFetch, + FeatureLz4, + FeatureDirectResults, + FeatureMultipleCatalog, + FeatureTracePropagation + }; + } + } +} diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ErrorEvent.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ErrorEvent.cs new file mode 100644 index 0000000000..7cd9194e41 --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/ErrorEvent.cs @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + /// + /// Tag definitions for Error events. + /// + internal static class ErrorEvent + { + public const string EventName = "Error"; + + // Error Classification + [TelemetryTag("error.type", ExportScope = TagExportScope.ExportAll, Required = true, Description = "Error type")] + public const string ErrorType = "error.type"; + + [TelemetryTag("http.status_code", ExportScope = TagExportScope.ExportAll, Description = "HTTP status code")] + public const string HttpStatusCode = "http.status_code"; + + [TelemetryTag("db.sql_state", ExportScope = TagExportScope.ExportAll, Description = "SQL state")] + public const string DbSqlState = "db.sql_state"; + + [TelemetryTag("error.operation", ExportScope = TagExportScope.ExportAll, Description = "Failed operation")] + public const string ErrorOperation = "error.operation"; + + [TelemetryTag("error.retried", ExportScope = TagExportScope.ExportAll, Description = "Was retried")] + public const string ErrorRetried = "error.retried"; + + [TelemetryTag("error.retry_count", ExportScope = TagExportScope.ExportAll, Description = "Retry count")] + public const string ErrorRetryCount = "error.retry_count"; + + [TelemetryTag("error.message", ExportScope = TagExportScope.ExportLocal, Description = "Error message")] + public const string ErrorMessage = "error.message"; + + [TelemetryTag("error.stack_trace", ExportScope = TagExportScope.ExportLocal, Description = "Stack trace")] + public const string ErrorStackTrace = "error.stack_trace"; + + /// + /// Returns tags allowed for Databricks export (privacy filter). + /// + public static IReadOnlyCollection GetDatabricksExportTags() + { + return new HashSet + { + ErrorType, + HttpStatusCode, + DbSqlState, + ErrorOperation, + ErrorRetried, + ErrorRetryCount + }; + } + } +} diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/StatementExecutionEvent.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/StatementExecutionEvent.cs new file mode 100644 index 0000000000..9998e94d3e --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/StatementExecutionEvent.cs @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + /// + /// Tag definitions for Statement execution events. + /// + internal static class StatementExecutionEvent + { + public const string EventName = "Statement.Execute"; + + // Identity + [TelemetryTag("statement.id", ExportScope = TagExportScope.ExportAll, Required = true, Description = "Statement ID")] + public const string StatementId = "statement.id"; + + [TelemetryTag("session.id", ExportScope = TagExportScope.ExportAll, Required = true, Description = "Session ID")] + public const string SessionId = "session.id"; + + // Result Metrics + [TelemetryTag("result.format", ExportScope = TagExportScope.ExportAll, Description = "Result format")] + public const string ResultFormat = "result.format"; + + [TelemetryTag("result.chunk_count", ExportScope = TagExportScope.ExportAll, Description = "Chunk count")] + public const string ResultChunkCount = "result.chunk_count"; + + [TelemetryTag("result.bytes_downloaded", ExportScope = TagExportScope.ExportAll, Description = "Bytes downloaded")] + public const string ResultBytesDownloaded = "result.bytes_downloaded"; + + [TelemetryTag("result.compression_enabled", ExportScope = TagExportScope.ExportAll, Description = "Compression enabled")] + public const string ResultCompressionEnabled = "result.compression_enabled"; + + [TelemetryTag("result.row_count", ExportScope = TagExportScope.ExportAll, Description = "Row count")] + public const string ResultRowCount = "result.row_count"; + + // Polling Metrics + [TelemetryTag("poll.count", ExportScope = TagExportScope.ExportAll, Description = "Poll count")] + public const string PollCount = "poll.count"; + + [TelemetryTag("poll.latency_ms", ExportScope = TagExportScope.ExportAll, Description = "Poll latency")] + public const string PollLatencyMs = "poll.latency_ms"; + + // Operation Type + [TelemetryTag("db.operation", ExportScope = TagExportScope.ExportAll, Description = "Operation type")] + public const string DbOperation = "db.operation"; + + [TelemetryTag("db.statement", ExportScope = TagExportScope.ExportLocal, Description = "SQL statement")] + public const string DbStatement = "db.statement"; + + [TelemetryTag("db.catalog", ExportScope = TagExportScope.ExportLocal, Description = "Catalog name")] + public const string DbCatalog = "db.catalog"; + + [TelemetryTag("db.schema", ExportScope = TagExportScope.ExportLocal, Description = "Schema name")] + public const string DbSchema = "db.schema"; + + public static IReadOnlyCollection GetDatabricksExportTags() + { + return new HashSet + { + StatementId, + SessionId, + ResultFormat, + ResultChunkCount, + ResultBytesDownloaded, + ResultCompressionEnabled, + ResultRowCount, + PollCount, + PollLatencyMs, + DbOperation + }; + } + } +} diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryEventType.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryEventType.cs new file mode 100644 index 0000000000..93ffc7b25b --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryEventType.cs @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + /// + /// Defines the types of telemetry events that can be emitted by the driver. + /// Each event type has its own set of allowed tags defined in corresponding *Event classes. + /// + public enum TelemetryEventType + { + /// + /// Connection open event. Emitted when a connection is established. + /// Tags defined in: + /// + ConnectionOpen, + + /// + /// Statement execution event. Emitted when a query or statement is executed. + /// Tags defined in: + /// + StatementExecution, + + /// + /// Error event. Emitted when an error occurs during any operation. + /// Tags defined in: + /// + Error + } +} diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTag.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTag.cs new file mode 100644 index 0000000000..f34e1bf405 --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTag.cs @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + /// + /// Controls where telemetry tags can be exported. + /// + [Flags] + internal enum TagExportScope + { + None = 0, + ExportLocal = 1, // Local diagnostics only + ExportDatabricks = 2, // Safe for Databricks service + ExportAll = ExportLocal | ExportDatabricks + } + + /// + /// Attribute for defining telemetry tags with export controls. + /// + [AttributeUsage(AttributeTargets.Field, AllowMultiple = false)] + internal sealed class TelemetryTagAttribute : Attribute + { + public string TagName { get; } + public TagExportScope ExportScope { get; set; } + public string? Description { get; set; } + public bool Required { get; set; } + + public TelemetryTagAttribute(string tagName) + { + TagName = tagName ?? throw new ArgumentNullException(nameof(tagName)); + ExportScope = TagExportScope.ExportLocal; + } + } +} diff --git a/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTagRegistry.cs b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTagRegistry.cs new file mode 100644 index 0000000000..130630fcee --- /dev/null +++ b/csharp/src/Drivers/Databricks/Telemetry/TagDefinitions/TelemetryTagRegistry.cs @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; + +namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions +{ + public static class TelemetryTagRegistry + { + /// + /// Gets tags allowed for Databricks export (privacy whitelist). + /// + // TODO: Explore alternate approaches to avoid maintaining separate GetDatabricksExportTags methods in each event class. + public static IReadOnlyCollection GetDatabricksExportTags(TelemetryEventType eventType) + { + return eventType switch + { + TelemetryEventType.ConnectionOpen => ConnectionOpenEvent.GetDatabricksExportTags(), + TelemetryEventType.StatementExecution => StatementExecutionEvent.GetDatabricksExportTags(), + TelemetryEventType.Error => ErrorEvent.GetDatabricksExportTags(), + _ => new HashSet() + }; + } + + /// + /// Checks if a tag should be exported to Databricks. + /// + public static bool ShouldExportToDatabricks(TelemetryEventType eventType, string tagName) + { + if (string.IsNullOrEmpty(tagName)) + { + return false; + } + + var allowedTags = GetDatabricksExportTags(eventType); + return allowedTags.Contains(tagName); + } + } +} diff --git a/csharp/test/Drivers/Databricks/Unit/Telemetry/TelemetryTagRegistryTests.cs b/csharp/test/Drivers/Databricks/Unit/Telemetry/TelemetryTagRegistryTests.cs new file mode 100644 index 0000000000..786015649b --- /dev/null +++ b/csharp/test/Drivers/Databricks/Unit/Telemetry/TelemetryTagRegistryTests.cs @@ -0,0 +1,52 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.TagDefinitions; +using Xunit; + +namespace Apache.Arrow.Adbc.Tests.Drivers.Databricks.Unit.Telemetry +{ + public class TelemetryTagRegistryTests + { + [Theory] + [InlineData(TelemetryEventType.StatementExecution, "db.statement")] + [InlineData(TelemetryEventType.ConnectionOpen, "server.address")] + [InlineData(TelemetryEventType.Error, "error.message")] + public void ShouldExportToDatabricks_SensitiveTags_ReturnsFalse(TelemetryEventType eventType, string tagName) + { + var shouldExport = TelemetryTagRegistry.ShouldExportToDatabricks(eventType, tagName); + Assert.False(shouldExport); + } + + [Theory] + [InlineData(TelemetryEventType.ConnectionOpen, "workspace.id")] + [InlineData(TelemetryEventType.StatementExecution, "result.chunk_count")] + [InlineData(TelemetryEventType.Error, "error.type")] + public void ShouldExportToDatabricks_NonSensitiveTags_ReturnsTrue(TelemetryEventType eventType, string tagName) + { + var shouldExport = TelemetryTagRegistry.ShouldExportToDatabricks(eventType, tagName); + Assert.True(shouldExport); + } + + [Fact] + public void GetDatabricksExportTags_UnknownEventType_ReturnsEmpty() + { + var tags = TelemetryTagRegistry.GetDatabricksExportTags((TelemetryEventType)999); + Assert.Empty(tags); + } + } +}