Add support for nonblocking custom BIO

Author: noteflakes

ext/openssl/ossl_ssl.c

@@ -38,7 +38,8 @@ static VALUE eSSLErrorWaitReadable;
 static VALUE eSSLErrorWaitWritable;
 
 static ID id_call, ID_callback_state, id_npn_protocols_encoded, id_each;
-static VALUE sym_exception, sym_wait_readable, sym_wait_writable;
+static VALUE sym_exception;
+VALUE sym_wait_readable, sym_wait_writable;
 
 static ID id_i_cert_store, id_i_ca_file, id_i_ca_path, id_i_verify_mode,
           id_i_verify_depth, id_i_verify_callback, id_i_client_ca,

ext/openssl/ossl_ssl_custom_bio.c

@@ -15,8 +15,18 @@
 #include "ruby/io/buffer.h"
 
 extern VALUE eSSLError;
+extern VALUE sym_wait_readable, sym_wait_writable;
 static ID id_read, id_write, id_call, id_eof_p;
 
+inline void bio_set_retry_state(BIO *bio, VALUE sym) {
+    if (sym == sym_wait_readable)
+        BIO_set_retry_read(bio);
+    else if (sym == sym_wait_writable)
+        BIO_set_retry_write(bio);
+    else
+        rb_raise(eSSLError, "Invalid return value from custom BIO");
+}
+
 static int
 ossl_ssl_custom_bio_in_read(BIO *bio, char *buf, int blen)
 {
@@ -37,6 +47,11 @@ ossl_ssl_custom_bio_in_read(BIO *bio, char *buf, int blen)
             VALUE buffer = rb_io_buffer_new(buf, blen, RB_IO_BUFFER_LOCKED);
             VALUE res = rb_funcall(read_proc, id_call, 2, buffer, INT2NUM(blen));
             rb_io_buffer_free_locked(buffer);
+
+            if (TYPE(res) == T_SYMBOL) {
+                bio_set_retry_state(bio, res);
+                return 0;
+            }
             return NUM2INT(res);
         }
         default:
@@ -63,6 +78,11 @@ ossl_ssl_custom_bio_out_write(BIO *bio, const char *buf, int blen)
             VALUE res = rb_funcall(write_proc, id_call, 2, buffer, INT2NUM(blen));
             RB_GC_GUARD(buffer);
             rb_io_buffer_free_locked(buffer);
+
+            if (TYPE(res) == T_SYMBOL) {
+                bio_set_retry_state(bio, res);
+                return 0;
+            }
             return NUM2INT(res);
         }
         default:

test/openssl/test_ssl.rb

@@ -2488,6 +2488,49 @@ def test_bio_method_custom_exception
       end
     }    
   end
+
+  def test_bio_method_custom_nonblock
+    omit_on_no_bio_method
+
+    start_server(ignore_listener_error: true) { |port|
+      begin
+        ops = []
+        ssl = OpenSSL::SSL::SSLSocket.open("127.0.0.1", port)
+        ssl.sync_close = true
+
+        io = ssl.to_io
+        read_proc = ->(buf, maxlen) {
+          ops << :read
+          ret = io.read_nonblock(maxlen, exception: false)
+          return ret if ret.is_a?(Symbol)
+
+          len = ret.bytesize
+          buf.set_string(ret)
+          len
+        }
+        write_proc = ->(buf, len) {
+          ops << :write
+          str = buf.get_string(0, len)
+          io.write_nonblock(str, exception: false)
+        }
+
+        ssl.bio_method = [read_proc, write_proc]
+
+       return_values = []
+        loop do
+          ret = ssl.connect_nonblock(exception: false)
+          return_values << ret
+          break if ret == ssl
+        end
+        
+        assert_equal [:write, :read], ops.uniq
+        assert_equal [:wait_readable, ssl], return_values.uniq
+      ensure
+        ssl&.close rescue nil
+      end
+    }
+  end
+
   def test_bio_method_invalid
     omit_on_no_bio_method