Implement Sigv4a in pure Ruby (#3071)

Author: alextwoods

Gemfile

@@ -22,6 +22,12 @@ unless defined?(JRUBY_VERSION)
   gem 'ox'
 end
 
+if defined?(JRUBY_VERSION)
+  # get the latest jruby-openssl to support sigv4a
+  # see: https://github.com/jruby/jruby-openssl/issues/30
+  gem 'jruby-openssl'
+end
+
 group :test do
   gem 'addressable'
   gem 'cucumber'

gems/aws-sigv4/CHANGELOG.md

@@ -1,6 +1,8 @@
 Unreleased Changes
 ------------------
 
+* Feature - Support `sigv4a` signing algorithm without `aws-crt`.
+
 1.8.0 (2023-11-28)
 ------------------
 

gems/aws-sigv4/lib/aws-sigv4.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'aws-sigv4/asymmetric_credentials'
 require_relative 'aws-sigv4/credentials'
 require_relative 'aws-sigv4/errors'
 require_relative 'aws-sigv4/signature'

gems/aws-sigv4/lib/aws-sigv4/asymmetric_credentials.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Aws
+  module Sigv4
+    # To make it easier to support mixed mode, we have created an asymmetric
+    # key derivation mechanism. This module derives
+    # asymmetric keys from the current secret for use with
+    # Asymmetric signatures.
+    # @api private
+    module AsymmetricCredentials
+
+      N_MINUS_2 = 0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551 - 2
+
+      # @param [String] :access_key_id
+      # @param [String] :secret_access_key
+      # @return [OpenSSL::PKey::EC, Hash]
+      def self.derive_asymmetric_key(access_key_id, secret_access_key)
+        check_openssl_support!
+        label = 'AWS4-ECDSA-P256-SHA256'
+        bit_len = 256
+        counter = 0x1
+        input_key = "AWS4A#{secret_access_key}"
+        d = 0 # d will end up being the private key
+        while true do
+
+          kdf_context = access_key_id.unpack('C*') + [counter].pack('C').unpack('C') #1 byte for counter
+          input = label.unpack('C*') + [0x00] + kdf_context + [bit_len].pack('L>').unpack('CCCC') # 4 bytes (change endianess)
+          k0 = OpenSSL::HMAC.digest("SHA256", input_key, ([0, 0, 0, 0x01] + input).pack('C*'))
+          c = be_bytes_to_num( k0.unpack('C*') )
+          if c <= N_MINUS_2
+            d = c + 1
+            break
+          elsif counter > 0xFF
+            raise 'Counter exceeded 1 byte - unable to get asym creds'
+          else
+            counter += 1
+          end
+        end
+
+        # compute the public key
+        group = OpenSSL::PKey::EC::Group.new('prime256v1')
+        public_key = group.generator.mul(d)
+
+        ec = generate_ec(public_key, d)
+
+        # pk_x and pk_y are not needed for signature, but useful in verification/testing
+        pk_b = public_key.to_octet_string(:uncompressed).unpack('C*') # 0x04 byte followed by 2 32-byte integers
+        pk_x = be_bytes_to_num(pk_b[1,32])
+        pk_y = be_bytes_to_num(pk_b[33,32])
+        [ec, {ec: ec, public_key: public_key, pk_x: pk_x, pk_y: pk_y, d: d}]
+      end
+
+      private
+
+      # @return [Number] The value of the bytes interpreted as a big-endian
+      # unsigned integer.
+      def self.be_bytes_to_num(bytes)
+        x = 0
+        bytes.each { |b| x = (x*256) + b }
+        x
+      end
+
+      # Prior to openssl3 we could directly set public and private key on EC
+      # However, openssl3 deprecated those methods and we must now construct
+      # a der with the keys and load the EC from it.
+      def self.generate_ec(public_key, d)
+        # format reversed from: OpenSSL::ASN1.decode_all(OpenSSL::PKey::EC.new.to_der)
+        asn1 = OpenSSL::ASN1::Sequence([
+          OpenSSL::ASN1::Integer(OpenSSL::BN.new(1)),
+          OpenSSL::ASN1::OctetString([d.to_s(16)].pack('H*')),
+          OpenSSL::ASN1::ASN1Data.new([OpenSSL::ASN1::ObjectId("prime256v1")], 0, :CONTEXT_SPECIFIC),
+          OpenSSL::ASN1::ASN1Data.new(
+            [OpenSSL::ASN1::BitString(public_key.to_octet_string(:uncompressed))],
+            1, :CONTEXT_SPECIFIC
+          )
+        ])
+        OpenSSL::PKey::EC.new(asn1.to_der)
+      end
+
+      def self.check_openssl_support!
+        return true unless defined?(JRUBY_VERSION)
+
+        # See: https://github.com/jruby/jruby-openssl/issues/306
+        # JRuby-openssl < 0.15 does not support OpenSSL::PKey::EC::Point#mul
+        return true if OpenSSL::PKey::EC::Point.instance_methods.include?(:mul)
+
+        raise 'Sigv4a Asymmetric Credential derivation requires jruby-openssl >= 0.15'
+      end
+    end
+  end
+end

gems/aws-sigv4/lib/aws-sigv4/signature.rb

@@ -32,6 +32,9 @@ def initialize(options)
       # @return [String] For debugging purposes.
       attr_accessor :content_sha256
 
+      # @return [String] For debugging purposes.
+      attr_accessor :signature
+
       # @return [Hash] Internal data for debugging purposes.
       attr_accessor :extra
     end

gems/aws-sigv4/lib/aws-sigv4/signer.rb

@@ -130,8 +130,7 @@ class Signer
       #   every other AWS service as of late 2016.
       #
       # @option options [Symbol] :signing_algorithm (:sigv4) The
-      #   algorithm to use for signing.  :sigv4a is only supported when
-      #   `aws-crt` is available.
+      #   algorithm to use for signing.
       #
       # @option options [Boolean] :omit_session_token (false)
       #   (Supported only when `aws-crt` is available) If `true`,
@@ -155,12 +154,6 @@ def initialize(options = {})
         @normalize_path = options.fetch(:normalize_path, true)
         @omit_session_token = options.fetch(:omit_session_token, false)
 
-        if @signing_algorithm == :sigv4a && !Signer.use_crt?
-          raise ArgumentError, 'You are attempting to sign a' \
-' request with sigv4a which requires the `aws-crt` gem.'\
-' Please install the gem or add it to your gemfile.'
-        end
-
         if @signing_algorithm == 'sigv4-s3express'.to_sym &&
            Signer.use_crt? && Aws::Crt::GEM_VERSION <= '0.1.9'
           raise ArgumentError,
@@ -249,6 +242,7 @@ def sign_request(request)
 
         http_method = extract_http_method(request)
         url = extract_url(request)
+        Signer.normalize_path(url) if @normalize_path
         headers = downcase_headers(request[:headers])
 
         datetime = headers['x-amz-date']
@@ -261,7 +255,7 @@ def sign_request(request)
         sigv4_headers = {}
         sigv4_headers['host'] = headers['host'] || host(url)
         sigv4_headers['x-amz-date'] = datetime
-        if creds.session_token
+        if creds.session_token && !@omit_session_token
           if @signing_algorithm == 'sigv4-s3express'.to_sym
             sigv4_headers['x-amz-s3session-token'] = creds.session_token
           else
@@ -271,26 +265,45 @@ def sign_request(request)
 
         sigv4_headers['x-amz-content-sha256'] ||= content_sha256 if @apply_checksum_header
 
+        if @signing_algorithm == :sigv4a && @region && !@region.empty?
+          sigv4_headers['x-amz-region-set'] = @region
+        end
         headers = headers.merge(sigv4_headers) # merge so we do not modify given headers hash
 
+        algorithm = sts_algorithm
+
         # compute signature parts
         creq = canonical_request(http_method, url, headers, content_sha256)
-        sts = string_to_sign(datetime, creq)
-        sig = signature(creds.secret_access_key, date, sts)
+        sts = string_to_sign(datetime, creq, algorithm)
+
+        sig =
+          if @signing_algorithm == :sigv4a
+            asymmetric_signature(creds, sts)
+          else
+            signature(creds.secret_access_key, date, sts)
+          end
+
+        algorithm = sts_algorithm
 
         # apply signature
         sigv4_headers['authorization'] = [
-          "AWS4-HMAC-SHA256 Credential=#{credential(creds, date)}",
+          "#{algorithm} Credential=#{credential(creds, date)}",
           "SignedHeaders=#{signed_headers(headers)}",
           "Signature=#{sig}",
         ].join(', ')
 
+        # skip signing the session token, but include it in the headers
+        if creds.session_token && @omit_session_token
+          sigv4_headers['x-amz-security-token'] = creds.session_token
+        end
+
         # Returning the signature components.
         Signature.new(
           headers: sigv4_headers,
           string_to_sign: sts,
           canonical_request: creq,
-          content_sha256: content_sha256
+          content_sha256: content_sha256,
+          signature: sig
         )
       end
 
@@ -424,6 +437,7 @@ def presign_url(options)
 
         http_method = extract_http_method(options)
         url = extract_url(options)
+        Signer.normalize_path(url) if @normalize_path
 
         headers = downcase_headers(options[:headers])
         headers['host'] ||= host(url)
@@ -436,8 +450,10 @@ def presign_url(options)
         content_sha256 ||= options[:body_digest]
         content_sha256 ||= sha256_hexdigest(options[:body] || '')
 
+        algorithm = sts_algorithm
+
         params = {}
-        params['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256'
+        params['X-Amz-Algorithm'] = algorithm
         params['X-Amz-Credential'] = credential(creds, date)
         params['X-Amz-Date'] = datetime
         params['X-Amz-Expires'] = presigned_url_expiration(options, expiration, Time.strptime(datetime, "%Y%m%dT%H%M%S%Z")).to_s
@@ -450,6 +466,10 @@ def presign_url(options)
         end
         params['X-Amz-SignedHeaders'] = signed_headers(headers)
 
+        if @signing_algorithm == :sigv4a && @region
+          params['X-Amz-Region-Set'] = @region
+        end
+
         params = params.map do |key, value|
           "#{uri_escape(key)}=#{uri_escape(value)}"
         end.join('&')
@@ -461,13 +481,23 @@ def presign_url(options)
         end
 
         creq = canonical_request(http_method, url, headers, content_sha256)
-        sts = string_to_sign(datetime, creq)
-        url.query += '&X-Amz-Signature=' + signature(creds.secret_access_key, date, sts)
+        sts = string_to_sign(datetime, creq, algorithm)
+        signature =
+          if @signing_algorithm == :sigv4a
+            asymmetric_signature(creds, sts)
+          else
+            signature(creds.secret_access_key, date, sts)
+          end
+        url.query += '&X-Amz-Signature=' + signature
         url
       end
 
       private
 
+      def sts_algorithm
+        @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256'
+      end
+
       def canonical_request(http_method, url, headers, content_sha256)
         [
           http_method,
@@ -479,9 +509,9 @@ def canonical_request(http_method, url, headers, content_sha256)
         ].join("\n")
       end
 
-      def string_to_sign(datetime, canonical_request)
+      def string_to_sign(datetime, canonical_request, algorithm)
         [
-          'AWS4-HMAC-SHA256',
+          algorithm,
           datetime,
           credential_scope(datetime[0,8]),
           sha256_hexdigest(canonical_request),
@@ -514,10 +544,10 @@ def event_string_to_sign(datetime, headers, payload, prior_signature, encoder)
       def credential_scope(date)
         [
           date,
-          @region,
+          (@region unless @signing_algorithm == :sigv4a),
           @service,
-          'aws4_request',
-        ].join('/')
+          'aws4_request'
+        ].compact.join('/')
       end
 
       def credential(credentials, date)
@@ -532,6 +562,16 @@ def signature(secret_access_key, date, string_to_sign)
         hexhmac(k_credentials, string_to_sign)
       end
 
+      def asymmetric_signature(creds, string_to_sign)
+        ec, _ = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key(
+          creds.access_key_id, creds.secret_access_key
+        )
+        sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign)
+        s = ec.dsa_sign_asn1(sts_digest)
+
+        Digest.hexencode(s)
+      end
+
       # Comparing to original signature v4 algorithm,
       # returned signature is a binary string instread of
       # hex-encoded string. (Since ':chunk-signature' requires
@@ -899,6 +939,18 @@ def uri_escape(string)
           end
         end
 
+        # @api private
+        def normalize_path(uri)
+          normalized_path = Pathname.new(uri.path).cleanpath.to_s
+          # Pathname is probably not correct to use. Empty paths will
+          # resolve to "." and should be disregarded
+          normalized_path = '' if normalized_path == '.'
+          # Ensure trailing slashes are correctly preserved
+          if uri.path.end_with?('/') && !normalized_path.end_with?('/')
+            normalized_path << '/'
+          end
+          uri.path = normalized_path
+        end
       end
     end
   end

gems/aws-sigv4/spec/asymmetric_credentials_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative 'spec_helper'
+
+module Aws
+  module Sigv4
+    describe AsymmetricCredentials do
+
+      # values for d,pk_x and pk_y are taken from get-vanilla sigv4a reference test
+      let(:access_key_id) { 'AKIDEXAMPLE' }
+      let(:secret_access_key) { 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' }
+      let(:ec) do
+        subject.derive_asymmetric_key(access_key_id, secret_access_key)[0]
+      end
+
+      let(:extra) do
+        subject.derive_asymmetric_key(access_key_id, secret_access_key)[1]
+      end
+
+      describe 'derive_asymmetric_key' do
+        it 'returns an EC PKey' do
+          expect(ec).to be_a(OpenSSL::PKey::EC)
+        end
+
+        it 'computes the private key' do
+          expect(extra[:d]).to be_a(Integer)
+          expect(extra[:d]).to eq 57437631014447175651096573782723065210935272504912550018654791361221980923292
+        end
+
+        it 'computes the public key' do
+          expect(extra[:public_key]).to be_a(OpenSSL::PKey::EC::Point)
+        end
+
+        it 'computes the pk_x and pk_y' do
+          expect(extra[:pk_x]).to be_a(Integer)
+          expect(extra[:pk_x]).to eq 82493312425604201858614910479538123276547530192671928569404457423490168469169
+
+          expect(extra[:pk_y]).to be_a(Integer)
+          expect(extra[:pk_y]).to eq 60777455846638291266199385583357715250110920888403467466325436560561456866584
+        end
+      end
+    end
+  end
+end