diff --git a/.github/workflows/ci-contrib.yml b/.github/workflows/ci-contrib.yml index 6bbc510cc6..4ff4d97cc4 100644 --- a/.github/workflows/ci-contrib.yml +++ b/.github/workflows/ci-contrib.yml @@ -27,6 +27,7 @@ jobs: - sql - mysql - sql-obfuscation + - sql-processor os: - ubuntu-latest name: "helpers-${{ matrix.gem }} / ${{ matrix.os }}" diff --git a/.toys/.data/releases.yml b/.toys/.data/releases.yml index 1182c128ac..c3f5bfba4f 100644 --- a/.toys/.data/releases.yml +++ b/.toys/.data/releases.yml @@ -55,6 +55,11 @@ gems: version_rb_path: lib/opentelemetry/helpers/sql_obfuscation/version.rb version_constant: [OpenTelemetry, Helpers, SqlObfuscation, VERSION] + - name: opentelemetry-helpers-sql-processor + directory: helpers/sql-processor + version_rb_path: lib/opentelemetry/helpers/sql_processor/version.rb + version_constant: [OpenTelemetry, Helpers, SqlProcessor, VERSION] + - name: opentelemetry-instrumentation-grape directory: instrumentation/grape version_constant: [OpenTelemetry, Instrumentation, Grape, VERSION] diff --git a/helpers/sql-processor/.rubocop.yml b/helpers/sql-processor/.rubocop.yml new file mode 100644 index 0000000000..4b31975de1 --- /dev/null +++ b/helpers/sql-processor/.rubocop.yml @@ -0,0 +1,4 @@ +inherit_from: ../../.rubocop.yml + +Gemspec/DevelopmentDependencies: + Enabled: false diff --git a/helpers/sql-processor/.yardopts b/helpers/sql-processor/.yardopts new file mode 100644 index 0000000000..6c38f31551 --- /dev/null +++ b/helpers/sql-processor/.yardopts @@ -0,0 +1,9 @@ +--no-private +--title=OpenTelemetry SQL Processor Instrumentation Helpers +--markup=markdown +--main=README.md +./lib/opentelemetry/helpers/**/*.rb +./lib/opentelemetry/helpers.rb +- +README.md +CHANGELOG.md diff --git a/helpers/sql-processor/CHANGELOG.md b/helpers/sql-processor/CHANGELOG.md new file mode 100644 index 0000000000..6e05cdd3d1 --- /dev/null +++ b/helpers/sql-processor/CHANGELOG.md @@ -0,0 +1 @@ +# Release History: opentelemetry-helpers-sql-processor diff --git a/helpers/sql-processor/Gemfile b/helpers/sql-processor/Gemfile new file mode 100644 index 0000000000..f77eaf288e --- /dev/null +++ b/helpers/sql-processor/Gemfile @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +source 'https://rubygems.org' + +gemspec + +group :test do + gem 'bundler', '~> 2.4' + gem 'minitest', '~> 5.0' + gem 'opentelemetry-test-helpers', '~> 0.3' + gem 'rake', '~> 13.0' + gem 'rubocop', '~> 1.79.1' + gem 'rubocop-performance', '~> 1.25.0' + gem 'simplecov', '~> 0.22.0' + gem 'yard', '~> 0.9' + gem 'yard-doctest', '~> 0.1.6' + if RUBY_VERSION >= '3.4' + gem 'base64' + gem 'mutex_m' + end +end diff --git a/helpers/sql-processor/LICENSE b/helpers/sql-processor/LICENSE new file mode 100644 index 0000000000..1ef7dad2c5 --- /dev/null +++ b/helpers/sql-processor/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The OpenTelemetry Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/helpers/sql-processor/README.md b/helpers/sql-processor/README.md new file mode 100644 index 0000000000..e869796009 --- /dev/null +++ b/helpers/sql-processor/README.md @@ -0,0 +1,65 @@ +# OpenTelemetry Instrumentation Helpers: SQL Processor + +This Ruby gem contains logic to process SQL, including obfuscation. It's intended for use by by gem authors instrumenting SQL adapter libraries, such as mysql2, pg, and trilogy. + +Obfuscation logic is largely drawn from the [New Relic Ruby agent's SQL Obfuscation Helpers module][new-relic-obfuscation-helpers]. + +## Usage + +Add the gem to your instrumentation's gemspec file: + +```ruby +# opentelemetry-instrumentation-your-gem.gemspec + spec.add_dependency 'opentelemetry-helpers-sql-processor' +``` + +Add the gem to your instrumentation's Gemfile: + +```ruby +# Gemfile + +group :test do + gem 'opentelemetry-helpers-sql-processor', path: '../../helpers/sql-processor' +end +``` +## Obfuscation + +Make sure the `Instrumentation` class for your gem contains configuration options for: + +- `:obfuscation_limit`: the length at which the SQL string will not be obfuscated + Example: `option :obfuscation_limit, default: 2000, validate: :integer` + +If you want to add support for a new adapter, update the following constants to include keys for your adapter: + +- `DIALECT_COMPONENTS` +- `CLEANUP_REGEX` + +You must also add a new constant that calls the `generate_regex` method with your adapter's DIALECT_COMPONENTS that is named like `_COMPONENTS_REGEX`, such as: `MYSQL_COMPONENTS_REGEX`. + +Check [New Relic's SQL Obfuscation Helpers module][new-relic-obfuscation-helpers] to see if regular expressions for your adapter already exist. + +### Examples + +To obfuscate sql in your library: + +```ruby +OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql, obfuscation_limit: config[:obfuscation_limit], adapter: :postgres) +``` + +## How can I get involved? + +The `opentelemetry-helpers-sql-processor` gem source is [on github][repo-github], along with related gems including `opentelemetry-instrumentation-pg` and `opentelemetry-instrumentation-trilogy`. + +The OpenTelemetry Ruby gems are maintained by the OpenTelemetry Ruby special interest group (SIG). You can get involved by joining us on our [GitHub Discussions][discussions-url], [Slack Channel][slack-channel] or attending our weekly meeting. See the [meeting calendar][community-meetings] for dates and times. For more information on this and other language SIGs, see the OpenTelemetry [community page][ruby-sig]. + +## License + +The `opentelemetry-helpers-sql-processor` gem is distributed under the Apache 2.0 license. See [LICENSE][license-github] for more information. + +[new-relic-obfuscation-helpers]: https://github.com/newrelic/newrelic-ruby-agent/blob/96e7aca22c1c873c0f5fe704a2b3bb19652db68e/lib/new_relic/agent/database/obfuscation_helpers.rb +[repo-github]: https://github.com/open-telemetry/opentelemetry-ruby +[license-github]: https://github.com/open-telemetry/opentelemetry-ruby-contrib/blob/main/LICENSE +[ruby-sig]: https://github.com/open-telemetry/community#ruby-sig +[community-meetings]: https://github.com/open-telemetry/community#community-meetings +[slack-channel]: https://cloud-native.slack.com/archives/C01NWKKMKMY +[discussions-url]: https://github.com/open-telemetry/opentelemetry-ruby/discussions diff --git a/helpers/sql-processor/Rakefile b/helpers/sql-processor/Rakefile new file mode 100644 index 0000000000..1a64ba842e --- /dev/null +++ b/helpers/sql-processor/Rakefile @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'bundler/gem_tasks' +require 'rake/testtask' +require 'yard' +require 'rubocop/rake_task' + +RuboCop::RakeTask.new + +Rake::TestTask.new :test do |t| + t.libs << 'test' + t.libs << 'lib' + t.test_files = FileList['test/**/*_test.rb'] +end + +YARD::Rake::YardocTask.new do |t| + t.stats_options = ['--list-undoc'] +end + +if RUBY_ENGINE == 'truffleruby' + task default: %i[test] +else + task default: %i[test rubocop yard] +end diff --git a/helpers/sql-processor/lib/opentelemetry-helpers-sql-processor.rb b/helpers/sql-processor/lib/opentelemetry-helpers-sql-processor.rb new file mode 100644 index 0000000000..a839b7e58d --- /dev/null +++ b/helpers/sql-processor/lib/opentelemetry-helpers-sql-processor.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'opentelemetry/helpers' diff --git a/helpers/sql-processor/lib/opentelemetry/helpers.rb b/helpers/sql-processor/lib/opentelemetry/helpers.rb new file mode 100644 index 0000000000..671d2b6e1a --- /dev/null +++ b/helpers/sql-processor/lib/opentelemetry/helpers.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'opentelemetry/helpers/sql_obfuscation' + +module OpenTelemetry + # The helpers module contains functionality shared across multiple + # instrumentation libraries + module Helpers + end +end diff --git a/helpers/sql-processor/lib/opentelemetry/helpers/sql_obfuscation.rb b/helpers/sql-processor/lib/opentelemetry/helpers/sql_obfuscation.rb new file mode 100644 index 0000000000..debb2b7e5c --- /dev/null +++ b/helpers/sql-processor/lib/opentelemetry/helpers/sql_obfuscation.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0module OpenTelemetry + +require 'opentelemetry-common' + +module OpenTelemetry + module Helpers + # + # This module contains SQL obfuscation behavior to share with + # instrumentation for specific database adapters. + # The class uses code from: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb + # + # To use this in your instrumentation, the `Instrumentation` class for + # your gem must contain configuration options for: + # * `:db_statement` + # Example: + # `option :db_statement, default: :include, validate: %I[omit include obfuscate]` + # * `:obfuscation_limit` + # Example: + # `option :obfuscation_limit, default: 2000, validate: :integer` + # + # If you want to add support for a new adapter, update the following + # constants to include keys for your adapter: + # * DIALECT_COMPONENTS + # * CLEANUP_REGEX + # You must also add a new constant that uses `generate_regex` with your + # adapter's dialect components that is named like + # `_COMPONENTS_REGEX`, such as: `MYSQL_COMPONENTS_REGEX`. + # + # @api public + module SqlObfuscation + module_function + + # From: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb + COMPONENTS_REGEX_MAP = { + single_quotes: /'(?:[^']|'')*?(?:\\'.*|'(?!'))/, + double_quotes: /"(?:[^"]|"")*?(?:\\".*|"(?!"))/, + dollar_quotes: /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/, + uuids: /\{?(?:[0-9a-fA-F]\-*){32}\}?/, + numeric_literals: /-?\b(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/, + boolean_literals: /\b(?:true|false|null)\b/i, + hexadecimal_literals: /0x[0-9a-fA-F]+/, + comments: /(?:#|--).*?(?=\r|\n|$)/i, + multi_line_comments: %r{(?:\/\*.*?\*\/)}m, + oracle_quoted_strings: /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/ + }.freeze + + DIALECT_COMPONENTS = { + default: COMPONENTS_REGEX_MAP.keys, + mysql: %i[single_quotes double_quotes numeric_literals boolean_literals + hexadecimal_literals comments multi_line_comments], + postgres: %i[single_quotes dollar_quotes uuids numeric_literals + boolean_literals comments multi_line_comments], + sqlite: %i[single_quotes numeric_literals boolean_literals hexadecimal_literals + comments multi_line_comments], + oracle: %i[single_quotes oracle_quoted_strings numeric_literals comments + multi_line_comments], + cassandra: %i[single_quotes uuids numeric_literals boolean_literals + hexadecimal_literals comments multi_line_comments] + }.freeze + + PLACEHOLDER = '?' + + # We use these to check whether the query contains any quote characters + # after obfuscation. If so, that's a good indication that the original + # query was malformed, and so our obfuscation can't reliably find + # literals. In such a case, we'll replace the entire query with a + # placeholder. + CLEANUP_REGEX = { + default: %r{'|"|\/\*|\*\/}, + mysql: %r{'|"|\/\*|\*\//}, + postgres: %r{'|\/\*|\*\/|\$(?!\?)/}, + sqlite: %r{'|\/\*|\*\//}, + cassandra: %r{'|\/\*|\*\//}, + oracle: %r{'|\/\*|\*\//} + }.freeze + + # @api private + def generate_regex(dialect) + components = DIALECT_COMPONENTS[dialect] + Regexp.union(components.map { |component| COMPONENTS_REGEX_MAP[component] }) + end + + DEFAULT_COMPONENTS_REGEX = generate_regex(:default) + MYSQL_COMPONENTS_REGEX = generate_regex(:mysql) + POSTGRES_COMPONENTS_REGEX = generate_regex(:postgres) + SQLITE_COMPONENTS_REGEX = generate_regex(:sqlite) + CASSANDRA_COMPONENTS_REGEX = generate_regex(:cassandra) + ORACLE_COMPONENTS_REGEX = generate_regex(:oracle) + + # This is a SQL obfuscation utility intended for use in database adapter instrumentation. + # + # @param sql [String] The SQL to obfuscate. + # @param obfuscation_limit [optional Integer] the length at which the SQL string will not be obfuscated + # @param adapter [optional Symbol] the type of database adapter calling the method. `:default`, `:mysql` and `:postgres` are supported. + # @return [String] The SQL query string where the values are replaced with "?". When the sql statement exceeds the obufscation limit + # the first matched pair from the SQL statement will be returned, with an appended truncation message. If trunaction is unsuccessful, + # a string describing the error will be returned. + # + # @api public + def obfuscate_sql(sql, obfuscation_limit: 2000, adapter: :default) + return "SQL not obfuscated, query exceeds #{obfuscation_limit} characters" if sql.size > obfuscation_limit + + regex = case adapter + when :mysql + MYSQL_COMPONENTS_REGEX + when :postgres + POSTGRES_COMPONENTS_REGEX + else + DEFAULT_COMPONENTS_REGEX + end + + # Original MySQL UTF-8 Encoding Fixes: + # https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/160 + # https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/345 + sql = OpenTelemetry::Common::Utilities.utf8_encode(sql, binary: true) + + sql = sql.gsub(regex, PLACEHOLDER) + return 'Failed to obfuscate SQL query - quote characters remained after obfuscation' if CLEANUP_REGEX[adapter].match(sql) + + sql + rescue StandardError => e + OpenTelemetry.handle_error(message: 'Failed to obfuscate SQL', exception: e) + end + end + end +end diff --git a/helpers/sql-processor/lib/opentelemetry/helpers/sql_processor/version.rb b/helpers/sql-processor/lib/opentelemetry/helpers/sql_processor/version.rb new file mode 100644 index 0000000000..825394bc6a --- /dev/null +++ b/helpers/sql-processor/lib/opentelemetry/helpers/sql_processor/version.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Helpers + module SqlProcessor + VERSION = '0.0.0' + end + end +end diff --git a/helpers/sql-processor/opentelemetry-helpers-sql-processor.gemspec b/helpers/sql-processor/opentelemetry-helpers-sql-processor.gemspec new file mode 100644 index 0000000000..dd3c039a6e --- /dev/null +++ b/helpers/sql-processor/opentelemetry-helpers-sql-processor.gemspec @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'opentelemetry/helpers/sql_processor/version' + +Gem::Specification.new do |spec| + spec.name = 'opentelemetry-helpers-sql-processor' + spec.version = OpenTelemetry::Helpers::SqlProcessor::VERSION + spec.authors = ['OpenTelemetry Authors'] + spec.email = ['cncf-opentelemetry-contributors@lists.cncf.io'] + + spec.summary = 'SQL Processing Instrumentation Helpers for the OpenTelemetry framework' + spec.description = 'SQL Processing Instrumentation Helpers for the OpenTelemetry framework' + spec.homepage = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib' + spec.license = 'Apache-2.0' + + spec.files = Dir.glob('lib/**/*.rb') + + Dir.glob('*.md') + + ['LICENSE', '.yardopts'] + spec.require_paths = ['lib'] + spec.required_ruby_version = ">= #{File.read(File.expand_path('../../gemspecs/RUBY_REQUIREMENT', __dir__))}" + + spec.add_dependency 'opentelemetry-common', '~> 0.21' + + if spec.respond_to?(:metadata) + spec.metadata['changelog_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}/file/CHANGELOG.md" + spec.metadata['source_code_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib/tree/main/helpers/sql-processor' + spec.metadata['bug_tracker_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib/issues' + spec.metadata['documentation_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}" + end + + spec.post_install_message = File.read(File.expand_path('../../gemspecs/POST_INSTALL_MESSAGE', __dir__)) +end diff --git a/helpers/sql-processor/test/fixtures/sql_obfuscation.json b/helpers/sql-processor/test/fixtures/sql_obfuscation.json new file mode 100644 index 0000000000..9c75b8f168 --- /dev/null +++ b/helpers/sql-processor/test/fixtures/sql_obfuscation.json @@ -0,0 +1,685 @@ +[ + { + "name": "back_quoted_identifiers.mysql", + "obfuscated": [ + "SELECT `t001`.`c2` FROM `t001` WHERE `t001`.`c2` = ? AND c3=? LIMIT ?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT `t001`.`c2` FROM `t001` WHERE `t001`.`c2` = 'value' AND c3=\"othervalue\" LIMIT ?" + }, + { + "name": "comment_delimiters_in_double_quoted_strings", + "obfuscated": [ + "SELECT * FROM t WHERE foo=? AND baz=?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT * FROM t WHERE foo=\"bar/*\" AND baz=\"whatever */qux\"" + }, + { + "name": "comment_delimiters_in_single_quoted_strings", + "obfuscated": [ + "SELECT * FROM t WHERE foo=? AND baz=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM t WHERE foo='bar/*' AND baz='whatever */qux'" + }, + { + "name": "double_quoted_identifiers.postgres", + "obfuscated": [ + "SELECT \"t001\".\"c2\" FROM \"t001\" WHERE \"t001\".\"c2\" = ? AND c3=? LIMIT ?" + ], + "dialects": [ + "postgres" + ], + "sql": "SELECT \"t001\".\"c2\" FROM \"t001\" WHERE \"t001\".\"c2\" = 'value' AND c3=1234 LIMIT 1" + }, + { + "name": "end_of_line_comment_in_double_quoted_string", + "obfuscated": [ + "SELECT * FROM t WHERE foo=? AND\n baz=?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT * FROM t WHERE foo=\"bar--\" AND\n baz=\"qux--\"" + }, + { + "name": "end_of_line_comment_in_single_quoted_string", + "obfuscated": [ + "SELECT * FROM t WHERE foo=? AND\n baz=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM t WHERE foo='bar--' AND\n baz='qux--'" + }, + { + "name": "end_of_query_comment_cstyle", + "obfuscated": [ + "SELECT * FROM foo WHERE bar=? ?", + "SELECT * FROM foo WHERE bar=? " + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM foo WHERE bar='baz' /* Hide Me */" + }, + { + "name": "end_of_query_comment_doubledash", + "obfuscated": [ + "SELECT * FROM foobar WHERE password=?\n?", + "SELECT * FROM foobar WHERE password=?\n" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM foobar WHERE password='secret2'\n-- No peeking!" + }, + { + "name": "end_of_query_comment_hash", + "obfuscated": [ + "SELECT foo, bar FROM baz WHERE password=? ?", + "SELECT foo, bar FROM baz WHERE password=? " + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT foo, bar FROM baz WHERE password='secret2' # Secret" + }, + { + "name": "escape_string_constants.postgres", + "sql": "SELECT \"col1\", \"col2\" from \"table\" WHERE \"col3\"=E'foo\\'bar\\\\baz' AND country=e'foo\\'bar\\\\baz'", + "obfuscated": [ + "SELECT \"col1\", \"col2\" from \"table\" WHERE \"col3\"=E?", + "SELECT \"col1\", \"col2\" from \"table\" WHERE \"col3\"=E? AND country=e?" + ], + "dialects": [ + "postgres" + ], + "comments": [ + "PostgreSQL supports an alternate string quoting mode where backslash escape", + "sequences are interpreted.", + "See: http://www.postgresql.org/docs/9.3/static/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-ESCAPE" + ] + }, + { + "name": "multiple_literal_types.mysql", + "obfuscated": [ + "INSERT INTO `X` values(?,?, ? , ?, ?)" + ], + "dialects": [ + "mysql" + ], + "sql": "INSERT INTO `X` values(\"test\",0, 1 , 2, 'test')" + }, + { + "name": "numbers_in_identifiers", + "obfuscated": [ + "SELECT c11.col1, c22.col2 FROM table c11, table c22 WHERE value=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT c11.col1, c22.col2 FROM table c11, table c22 WHERE value='nothing'" + }, + { + "name": "numeric_literals", + "sql": "INSERT INTO X VALUES(1, 23456, 123.456, 99+100)", + "obfuscated": [ + "INSERT INTO X VALUES(?, ?, ?, ?+?)", + "INSERT INTO X VALUES(?, ?, ?.?, ?+?)" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "string_double_quoted.mysql", + "obfuscated": [ + "SELECT * FROM table WHERE name=? AND value=?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT * FROM table WHERE name=\"foo\" AND value=\"don't\"" + }, + { + "name": "string_single_quoted", + "obfuscated": [ + "SELECT * FROM table WHERE name=? AND value = ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM table WHERE name='foo' AND value = 'bar'" + }, + { + "name": "string_with_backslash_and_twin_single_quotes", + "obfuscated": [ + "SELECT * FROM table WHERE col=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM table WHERE col='foo\\''bar'", + "comments": [ + "If backslashes are being ignored in single-quoted strings", + "(standard_conforming_strings=on in PostgreSQL, or NO_BACKSLASH_ESCAPES is on", + "in MySQL), then this is valid SQL." + ] + }, + { + "name": "string_with_embedded_double_quote", + "obfuscated": [ + "SELECT * FROM table WHERE col1=? AND col2=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM table WHERE col1='foo\"bar' AND col2='what\"ever'" + }, + { + "name": "string_with_embedded_newline", + "obfuscated": [ + "select * from accounts where accounts.name != ? order by accounts.name" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "select * from accounts where accounts.name != 'dude \n newline' order by accounts.name" + }, + { + "name": "string_with_embedded_single_quote.mysql", + "obfuscated": [ + "SELECT * FROM table WHERE col1=? AND col2=?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT * FROM table WHERE col1=\"don't\" AND col2=\"won't\"" + }, + { + "name": "string_with_escaped_quotes.mysql", + "sql": "INSERT INTO X values('', 'jim''s ssn',0, 1 , 'jim''s son''s son', \"\"\"jim''s\"\" hat\", \"\\\"jim''s secret\\\"\")", + "obfuscated": [ + "INSERT INTO X values(?, ?,?, ? , ?, ?, ?", + "INSERT INTO X values(?, ?,?, ? , ?, ?, ?)" + ], + "dialects": [ + "mysql" + ] + }, + { + "name": "string_with_trailing_backslash", + "sql": "SELECT * FROM table WHERE name='foo\\' AND color='blue'", + "obfuscated": [ + "SELECT * FROM table WHERE name=?", + "SELECT * FROM table WHERE name=? AND color=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "comments": [ + "If backslashes are being ignored in single-quoted strings", + "(standard_conforming_strings=on in PostgreSQL, or NO_BACKSLASH_ESCAPES is on", + "in MySQL), then this is valid SQL." + ] + }, + { + "name": "string_with_trailing_escaped_backslash.mysql", + "obfuscated": [ + "SELECT * FROM table WHERE foo=?" + ], + "dialects": [ + "mysql" + ], + "sql": "SELECT * FROM table WHERE foo=\"this string ends with a backslash\\\\\"" + }, + { + "name": "string_with_trailing_escaped_backslash_single_quoted", + "obfuscated": [ + "SELECT * FROM table WHERE foo=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "SELECT * FROM table WHERE foo='this string ends with a backslash\\\\'" + }, + { + "name": "string_with_trailing_escaped_quote", + "sql": "SELECT * FROM table WHERE name='foo\\'' AND color='blue'", + "obfuscated": [ + "SELECT * FROM table WHERE name=?", + "SELECT * FROM table WHERE name=? AND color=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "string_with_twin_single_quotes", + "obfuscated": [ + "INSERT INTO X values(?, ?,?, ? , ?)" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "sql": "INSERT INTO X values('', 'a''b c',0, 1 , 'd''e f''s h')" + }, + { + "name": "end_of_line_comments_with_quotes", + "sql": "SELECT * FROM t WHERE -- '\n bar='baz' -- '", + "obfuscated": [ + "SELECT * FROM t WHERE ?\n bar=? ?", + "SELECT * FROM t WHERE ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "mixed_comments_and_quotes", + "sql": "SELECT * FROM t WHERE /* ' */ \n bar='baz' -- '", + "obfuscated": [ + "SELECT * FROM t WHERE ? \n bar=? ?", + "SELECT * FROM t WHERE ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "mixed_quotes_comments_and_newlines", + "sql": "SELECT * FROM t WHERE -- '\n /* ' */ c2='xxx' /* ' */\n c='x\n xx' -- '", + "obfuscated": [ + "SELECT * FROM t WHERE ?\n ? c2=? ?\n c=? ?", + "SELECT * FROM t WHERE ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "mixed_quotes_end_of_line_comments", + "sql": "SELECT * FROM t WHERE -- '\n c='x\n xx' -- '", + "obfuscated": [ + "SELECT * FROM t WHERE ?\n c=? ?", + "SELECT * FROM t WHERE ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "quote_delimiters_in_comments", + "sql": "SELECT * FROM foo WHERE col='value1' AND /* don't */ col2='value1' /* won't */", + "obfuscated": [ + "SELECT * FROM foo WHERE col=? AND ? col2=? ?", + "SELECT * FROM foo WHERE col=? AND ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "malformed/unterminated_double_quoted_string.mysql", + "sql": "SELECT * FROM table WHERE foo='bar' AND baz=\"nothing to see here'", + "dialects": [ + "mysql" + ], + "obfuscated": [ + "?" + ], + "malformed": true + }, + { + "name": "malformed/unterminated_single_quoted_string", + "sql": "SELECT * FROM table WHERE foo='bar' AND baz='nothing to see here", + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ], + "obfuscated": [ + "?" + ], + "malformed": true + }, + { + "name": "dollar_quotes", + "sql": "SELECT * FROM \"foo\" WHERE \"foo\" = $a$dollar quotes can be $b$nested$b$$a$ and bar = 'baz'", + "obfuscated": [ + "SELECT * FROM \"foo\" WHERE \"foo\" = ? and bar = ?" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "variable_substitution_not_mistaken_for_dollar_quotes", + "sql": "INSERT INTO \"foo\" (\"bar\", \"baz\", \"qux\") VALUES ($1, $2, $3) RETURNING \"id\"", + "obfuscated": [ + "INSERT INTO \"foo\" (\"bar\", \"baz\", \"qux\") VALUES ($?, $?, $?) RETURNING \"id\"" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "non_quote_escape", + "sql": "select * from foo where bar = 'some\\tthing' and baz = 10", + "obfuscated": [ + "select * from foo where bar = ? and baz = ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "end_of_string_backslash_and_line_comment_with_quite", + "sql": "select * from users where user = 'user1\\' password = 'secret 2' -- ->don't count this quote", + "obfuscated": [ + "select * from users where user = ?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "oracle_bracket_quote", + "sql": "select * from foo where bar=q'[baz's]' and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "oracle" + ] + }, + { + "name": "oracle_brace_quote", + "sql": "select * from foo where bar=q'{baz's}' and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "oracle" + ] + }, + { + "name": "oracle_angle_quote", + "sql": "select * from foo where bar=q'' and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "oracle" + ] + }, + { + "name": "oracle_paren_quote", + "sql": "select * from foo where bar=q'(baz's)' and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "oracle" + ] + }, + { + "name": "cassandra_blobs", + "sql": "select * from foo where bar=0xabcdef123 and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "cassandra", + "sqlite" + ] + }, + { + "name": "hex_literals", + "sql": "select * from foo where bar=0x2F and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "mysql", + "cassandra", + "sqlite" + ] + }, + { + "name": "exponential_literals", + "sql": "select * from foo where bar=1.234e-5 and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "negative_integer_literals", + "sql": "select * from foo where bar=-1.234e-5 and x=-5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra", + "sqlite" + ] + }, + { + "name": "uuid", + "sql": "select * from foo where bar=01234567-89ab-cdef-0123-456789abcdef and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "postgres", + "cassandra" + ] + }, + { + "name": "uuid_with_braces", + "sql": "select * from foo where bar={01234567-89ab-cdef-0123-456789abcdef} and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "uuid_no_dashes", + "sql": "select * from foo where bar=0123456789abcdef0123456789abcdef and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "uuid_random_dashes", + "sql": "select * from foo where bar={012-345678-9abc-def012345678-9abcdef} and x=5", + "obfuscated": [ + "select * from foo where bar=? and x=?" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "booleans", + "sql": "select * from truestory where bar=true and x=FALSE", + "obfuscated": [ + "select * from truestory where bar=? and x=?" + ], + "dialects": [ + "mysql", + "postgres", + "cassandra", + "sqlite" + ] + }, + { + "name": "in_clause_digits", + "sql": "select * from foo where bar IN (123, 456, 789)", + "obfuscated": [ + "select * from foo where bar IN (?, ?, ?)" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra" + ] + }, + { + "name": "in_clause_strings", + "sql": "select * from foo where bar IN ('asdf', 'fdsa')", + "obfuscated": [ + "select * from foo where bar IN (?, ?)" + ], + "dialects": [ + "mysql", + "postgres", + "oracle", + "cassandra" + ] + }, + { + "name": "prepended_comments_with_quotes.postgres", + "sql": "/*application:Demo,controller:posts,action:update*/ UPDATE \"posts\" SET \"updated_at\" = '2023-11-01 19:02:34.795909' WHERE \"posts\".\"id\" = 3", + "obfuscated": [ + "? UPDATE \"posts\" SET \"updated_at\" = ? WHERE \"posts\".\"id\" = ?" + ], + "dialects": [ + "postgres" + ] + }, + { + "name": "prepended_comments_with_quotes.mysql", + "sql": "/*action='show',application='TrilogyTest',controller='users'*/ SELECT `users`.* FROM `users` WHERE `users`.`id` = 1 LIMIT 1", + "obfuscated": [ + "? SELECT `users`.* FROM `users` WHERE `users`.`id` = ? LIMIT ?" + ], + "dialects": [ + "mysql" + ] + }, + { + "name": "prepended_multiline_comments_with_quotes.mysql", + "sql": "/*action='show',\napplication='TrilogyTest',controller='users'*/\nSELECT `users`.*\nFROM `users`\nWHERE `users`.`id` = 1 LIMIT 1", + "obfuscated": [ + "?\nSELECT `users`.*\nFROM `users`\nWHERE `users`.`id` = ? LIMIT ?" + ], + "dialects": [ + "mysql" + ] + } +] diff --git a/helpers/sql-processor/test/helpers/sql_obfuscation_test.rb b/helpers/sql-processor/test/helpers/sql_obfuscation_test.rb new file mode 100644 index 0000000000..39229d47f1 --- /dev/null +++ b/helpers/sql-processor/test/helpers/sql_obfuscation_test.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# This file is distributed under New Relic's license terms. +# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details. + +require_relative '../test_helper' + +class SqlObfuscationTest < Minitest::Test + def test_named_arg_defaults_obfuscates + sql = "SELECT * from users where users.id = 1 and users.email = 'test@test.com'" + expected = 'SELECT * from users where users.id = ? and users.email = ?' + result = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql) + + assert_equal(expected, result) + end + + def test_obfuscation_returns_message_when_limit_is_reached + sql = "SELECT * from users where users.id = 1 and users.email = 'test@test.com'" + expected = 'SQL not obfuscated, query exceeds 42 characters' + result = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql, obfuscation_limit: 42) + + assert_equal(expected, result) + end + + def test_non_utf_8_encoded_string_obfuscates_with_mysql + sql = "SELECT * from users where users.id = 1 and users.email = 'test@test.com\255'" + expected = 'SELECT * from users where users.id = ? and users.email = ?' + result = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql, adapter: :mysql) + + assert_equal(expected, result) + end + + def test_non_utf_8_encoded_string_obfuscates_with_postgres + sql = "SELECT * from users where users.id = 1 and users.email = 'test@test.com\255'" + expected = 'SELECT * from users where users.id = ? and users.email = ?' + result = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql, adapter: :postgres) + + assert_equal(expected, result) + end + + def test_statement_with_emoji_encodes_utf_8_and_obfuscates + sql = "SELECT * from users where users.id = 1 and users.email = 'test@😄.com'" + expected = 'SELECT * from users where users.id = ? and users.email = ?' + result = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(sql) + + assert_equal(expected, result) + end + + # The following tests and their corresponding fixture are based on code from + # the New Relic Ruby agent. + # source: https://github.com/newrelic/newrelic-ruby-agent/blob/cb72bb5fab3fb318613421c86863a5ccdd2ff250/test/new_relic/agent/database/sql_obfuscation_test.rb + + FAILED_TO_OBFUSCATE_MESSAGE = 'Failed to obfuscate SQL query - quote characters remained after obfuscation' + + def build_failure_message(statement, dialect, acceptable_outputs, actual_output) + msg = "Failed to obfuscate #{dialect} query correctly.\n" + msg << "Input: #{statement}\n" + if acceptable_outputs.size == 1 + msg << "Expected: #{acceptable_outputs.first}\n" + else + msg << "Acceptable outputs:\n" + acceptable_outputs.each do |output| + msg << " #{output}\n" + end + end + msg << "Actual: #{actual_output}\n" + msg + end + + def self.load_fixture + data = File.read("#{Dir.pwd}/test/fixtures/sql_obfuscation.json") + JSON.parse(data) + end + + load_fixture.each do |test_case| + name = test_case['name'] + query = test_case['sql'] + acceptable_outputs = test_case['obfuscated'] + dialects = test_case['dialects'] + + # If the entire query is obfuscated because it's malformed, we use a + # placeholder message instead of just '?', so add that to the acceptable + # outputs. + acceptable_outputs << FAILED_TO_OBFUSCATE_MESSAGE if test_case['malformed'] + + dialects.each do |dialect| + define_method(:"test_sql_obfuscation_#{name}_#{dialect}") do + actual_obfuscated = OpenTelemetry::Helpers::SqlObfuscation.obfuscate_sql(query, adapter: dialect.to_sym) + message = build_failure_message(query, dialect, acceptable_outputs, actual_obfuscated) + + assert_includes(acceptable_outputs, actual_obfuscated, message) + end + end + end + ## End New Relic tests +end diff --git a/helpers/sql-processor/test/test_helper.rb b/helpers/sql-processor/test/test_helper.rb new file mode 100644 index 0000000000..419b09eb04 --- /dev/null +++ b/helpers/sql-processor/test/test_helper.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'simplecov' +require 'bundler/setup' +Bundler.require(:default, :development, :test) + +require 'minitest/autorun' +require 'opentelemetry-helpers-sql-processor'