Merge branch 'master-2.2' into dist/2.2/xenial

Author: sorah

ChangeLog

@@ -1,3 +1,242 @@
+Wed Mar 28 23:48:24 2018  Eric Wong  <[email protected]>
+
+	webrick: prevent response splitting and header injection
+
+	Original patch by tenderlove (with minor style adjustments).
+
+	* lib/webrick/httpresponse.rb (send_header): call check_header
+	  (check_header): raise on embedded CRLF in header value
+	* test/webrick/test_httpresponse.rb
+	  (test_prevent_response_splitting_headers): new test
+	* (test_prevent_response_splitting_cookie_headers): ditto
+
+Wed Mar 28 23:45:36 2018  Eric Wong  <[email protected]>
+
+	webrick: use IO.copy_stream for multipart response
+
+	Use the new Proc response body feature to generate a multipart
+	range response dynamically.  We use a flat array to minimize
+	object overhead as much as possible; as many ranges may fit
+	into an HTTP request header.
+
+	* lib/webrick/httpservlet/filehandler.rb (multipart_body): new method
+	  (make_partial_content): use multipart_body
+
+	webrick/httprequest: limit request headers size
+
+	We use the same 112 KB limit started (AFAIK) by Mongrel, Thin,
+	and Puma to prevent malicious users from using up all the memory
+	with a single request.  This also limits the damage done by
+	excessive ranges in multipart Range: requests.
+
+	Due to the way we rely on IO#gets and the desire to keep
+	the code simple, the actual maximum header may be 4093 bytes
+	larger than 112 KB, but we're splitting hairs at that point.
+
+	* lib/webrick/httprequest.rb: define MAX_HEADER_LENGTH
+	  (read_header): raise when headers exceed max length
+
+	webrick/httpservlet/cgihandler: reduce memory use
+
+	WEBrick::HTTPRequest#body can be passed a block to process the
+	body in chunks.  Use this feature to avoid building a giant
+	string in memory.
+
+	* lib/webrick/httpservlet/cgihandler.rb (do_GET):
+	  avoid reading entire request body into memory
+	  (do_POST is aliased to do_GET, so it handles bodies)
+
+	webrick/httprequest: raise correct exception
+
+	"BadRequest" alone does not resolve correctly, it is in the
+	HTTPStatus namespace.
+
+	* lib/webrick/httprequest.rb (read_chunked): use correct exception
+	* test/webrick/test_httpserver.rb (test_eof_in_chunk): new test
+
+	webrick/httprequest: use InputBufferSize for chunked requests
+
+	While WEBrick::HTTPRequest#body provides a Proc interface
+	for streaming large request bodies, clients must not force
+	the server to use an excessively large chunk size.
+
+	* lib/webrick/httprequest.rb (read_chunk_size): limit each
+	  read and block.call to :InputBufferSize in config.
+	* test/webrick/test_httpserver.rb (test_big_chunks): new test
+
+	webrick: add test for Digest auth-int
+
+	No changes to the actual code, this is a new test for
+	a feature for which no tests existed.  I don't understand
+	the Digest authentication code well at all, but this is
+	necessary for the subsequent change.
+
+	* test/webrick/test_httpauth.rb (test_digest_auth_int): new test
+	  (credentials_for_request): support bodies with POST
+
+	webrick/httpauth/digestauth: stream req.body
+
+	WARNING! WARNING! WARNING!  LIKELY BROKEN CHANGE
+
+	Pass a proc to WEBrick::HTTPRequest#body to avoid reading a
+	potentially large request body into memory during
+	authentication.
+
+	WARNING! this will break apps completely which want to do
+	something with the body besides calculating the MD5 digest
+	of it.
+
+	Also, keep in mind that probably nobody uses "auth-int".
+	Servers such as Apache, lighttpd, nginx don't seem to
+	support it; nor does curl when using POST/PUT bodies;
+	and we didn't have tests for it until now...
+
+	* lib/webrick/httpauth/digestauth.rb (_authenticate): stream req.body
+
+Wed Mar 28 23:41:53 2018  NAKAMURA Usaku  <[email protected]>
+
+	get rid of test error/failure on Windows introduced at r62955
+
+	* lib/webrick/httpresponse.rb (send_body_io): use seek if
+	  NotImplementedError is raised in IO.copy_stream with offset.
+
+	* lib/webrick/httpservlet/filehandler.rb (multipart_body): ditto.
+
+Wed Mar 28 23:41:53 2018  Eric Wong  <[email protected]>
+
+	webrick: support Proc objects as body responses
+
+	* lib/webrick/httpresponse.rb (send_body): call send_body_proc
+	  (send_body_proc): new method
+	  (class ChunkedWrapper): new class
+
+	* test/webrick/test_httpresponse.rb (test_send_body_proc): new test
+	  (test_send_body_proc_chunked): ditto
+	  [Feature #855]
+
+	webrick: favor .write over << method
+
+	This will make the next change to use IO.copy_stream
+	easier-to-read.  When we can drop Ruby 2.4 support in a few
+	years, this will allow us to use writev(2) with multiple
+	arguments for headers and chunked responses.
+
+	* lib/webrick/cgi.rb (write): new wrapper method
+	  lib/webrick/httpresponse.rb: (send_header): use socket.write
+	  (send_body_io): ditto
+	  (send_body_string): ditto
+	  (send_body_proc): ditto
+	  (_write_data): ditto
+	  (ChunkedWrapper#write): ditto
+	  (_send_file): ditto
+
+	webrick/httpresponse: IO.copy_stream for regular files
+
+	Remove the redundant _send_file method since its functionality
+	is unnecessary with IO.copy_stream.  IO.copy_stream also allows
+	the use of sendfile under some OSes to speed up copies to
+	non-TLS sockets.
+
+	Testing with "curl >/dev/null" and "ruby -run -e httpd" to
+	read a 1G file over Linux loopback reveals a reduction from
+	around ~0.770 to ~0.490 seconds on the client side.
+
+	* lib/webrick/httpresponse.rb (send_body_io): use IO.copy_stream
+	  (_send_file): remove
+	  [Feature #14237]
+
+	webrick: use IO.copy_stream for single range response
+
+	This is also compatible with range responses generated
+	by Rack::File (tested with rack 2.0.3).
+
+	* lib/webrick/httpresponse.rb (send_body_io): use Content-Range
+	* lib/webrick/httpservlet/filehandler.rb (make_partial_content):
+	  use File object for the single range case
+	* test/webrick/test_filehandler.rb (get_res_body): use send_body
+	  to test result
+
+	test/webrick/test_filehandler.rb: stricter multipart range test
+
+	We need to ensure we generate compatibile output in
+	the face of future changes
+
+	* test/webrick/test_filehandler.rb (test_make_partial_content):
+	  check response body
+
+	webrick: quiet warning for multi-part ranges
+
+	Content-Length is ignored by WEBrick::HTTPResponse even if we
+	calculate it, so instead we chunk responses to HTTP/1.1 clients
+	and terminate HTTP/1.0 connections.
+
+	* lib/webrick/httpservlet/filehandler.rb (make_partial_content):
+	  quiet warning
+
+	webrick/httpresponse: make ChunkedWrapper copy_stream-compatible
+
+	The .write method needs to return the number of bytes written
+	to avoid confusing IO.copy_stream.
+
+	* lib/webrick/httpresponse.rb (ChunkedWrapper#write): return bytes written
+	  (ChunkedWrapper#<<): return self
+
+	webrick: use IO.copy_stream for multipart response
+
+	Use the new Proc response body feature to generate a multipart
+	range response dynamically.  We use a flat array to minimize
+	object overhead as much as possible; as many ranges may fit
+	into an HTTP request header.
+
+	* lib/webrick/httpservlet/filehandler.rb (multipart_body): new method
+	  (make_partial_content): use multipart_body
+
+Wed Mar 28 23:37:18 2018  Nobuyoshi Nakada  <[email protected]>
+
+	pack.c: fix underflow
+
+	* pack.c (pack_unpack_internal): get rid of underflow.
+	  https://hackerone.com/reports/298246
+
+Wed Mar 28 23:35:28 2018  Nobuyoshi Nakada  <[email protected]>
+
+	unixsocket.c: check NUL bytes
+
+	* ext/socket/unixsocket.c (rsock_init_unixsock): check NUL bytes.
+	  https://hackerone.com/reports/302997
+
+	unixsocket.c: abstract namespace
+
+	* ext/socket/unixsocket.c (unixsock_path_value): fix r62991 for
+	  Linux abstract namespace.
+
+Wed Mar 28 23:30:32 2018  SHIBATA Hiroshi  <[email protected]>
+
+	Ignore file separator from tmpfile/tmpdir name.
+
+Wed Mar 28 23:27:23 2018  Nobuyoshi Nakada  <[email protected]>
+
+	dir.c: check NUL bytes
+
+	* dir.c (GlobPathValue): should be used in rb_push_glob only.
+	  other methods should use FilePathValue.
+	  https://hackerone.com/reports/302338
+
+	* dir.c (rb_push_glob): expand GlobPathValue
+
+Sat Feb 17 01:24:49 2018  SHIBATA Hiroshi  <[email protected]>
+
+	Merge RubyGems 2.7.6 from upstream.
+
+	It fixed some security vulnerabilities.
+
+	http://blog.rubygems.org/2018/02/15/2.7.6-released.html
+
+	fix regexp literal warning.
+
+	* test/rubygems/test_gem_server.rb: eliminate duplicated character class warning.
+	  [Bug #14481]
+
 Fri Dec 15 00:08:26 2017  NAKAMURA Usaku  <[email protected]>
 
 	* test/net/ftp/test_ftp.rb (process_port_or_eprt): merge a part of

debian/changelog

@@ -1,3 +1,9 @@
+ruby2.2 (2.2.10-0nkmi1) unstable; urgency=medium
+
+  * Ruby 2.2.10
+
+ -- Sorah Fukumori <[email protected]>  Thu, 29 Mar 2018 00:10:42 +0000
+
 ruby2.2 (2.2.9-0nkmi1~xenial) xenial; urgency=medium
 
   * new upstream version

dir.c

@@ -423,15 +423,6 @@ static const rb_data_type_t dir_data_type = {
 
 static VALUE dir_close(VALUE);
 
-#define GlobPathValue(str, safe) \
-    /* can contain null bytes as separators */	\
-    (!RB_TYPE_P((str), T_STRING) ?		\
-     (void)FilePathValue(str) :			\
-     (void)(check_safe_glob((str), (safe)),		\
-      check_glob_encoding(str), (str)))
-#define check_safe_glob(str, safe) ((safe) ? rb_check_safe_obj(str) : (void)0)
-#define check_glob_encoding(str) rb_enc_check((str), rb_enc_from_encoding(rb_usascii_encoding()))
-
 static VALUE
 dir_s_alloc(VALUE klass)
 {
@@ -480,7 +471,7 @@ dir_initialize(int argc, VALUE *argv, VALUE dir)
 	}
     }
 
-    GlobPathValue(dirname, FALSE);
+    FilePathValue(dirname);
     orig = rb_str_dup_frozen(dirname);
     dirname = rb_str_encode_ospath(dirname);
     dirname = rb_str_dup_frozen(dirname);
@@ -2050,7 +2041,14 @@ rb_push_glob(VALUE str, int flags) /* '\0' is delimiter */
     long offset = 0;
     VALUE ary;
 
-    GlobPathValue(str, TRUE);
+    /* can contain null bytes as separators */
+    if (!RB_TYPE_P((str), T_STRING)) {
+	FilePathValue(str);
+    }
+    else {
+	rb_check_safe_obj(str);
+	rb_enc_check(str, rb_enc_from_encoding(rb_usascii_encoding()));
+    }
     ary = rb_ary_new();
 
     while (offset < RSTRING_LEN(str)) {
@@ -2080,7 +2078,7 @@ dir_globs(long argc, const VALUE *argv, int flags)
     for (i = 0; i < argc; ++i) {
 	int status;
 	VALUE str = argv[i];
-	GlobPathValue(str, TRUE);
+	FilePathValue(str);
 	status = push_glob(ary, str, flags);
 	if (status) GLOB_JUMP_TAG(status);
     }

ext/socket/unixsocket.c

@@ -25,6 +25,28 @@ unixsock_connect_internal(VALUE a)
 			        arg->sockaddrlen, 0);
 }
 
+static VALUE
+unixsock_path_value(VALUE path)
+{
+#ifdef __linux__
+#define TO_STR_FOR_LINUX_ABSTRACT_NAMESPACE 0
+
+    VALUE name = path;
+#if TO_STR_FOR_LINUX_ABSTRACT_NAMESPACE
+    const int isstr = !NIL_P(name = rb_check_string_type(name));
+#else
+    const int isstr = RB_TYPE_P(name, T_STRING);
+#endif
+    if (isstr) {
+        if (RSTRING_LEN(name) == 0 || RSTRING_PTR(name)[0] == '\0') {
+            rb_check_safe_obj(name);
+            return name;             /* ignore encoding */
+        }
+    }
+#endif
+    return rb_get_path(path);
+}
+
 VALUE
 rsock_init_unixsock(VALUE sock, VALUE path, int server)
 {
@@ -33,7 +55,7 @@ rsock_init_unixsock(VALUE sock, VALUE path, int server)
     int fd, status;
     rb_io_t *fptr;
 
-    SafeStringValue(path);
+    path = unixsock_path_value(path);
 
     INIT_SOCKADDR_UN(&sockaddr, sizeof(struct sockaddr_un));
     if (sizeof(sockaddr.sun_path) < (size_t)RSTRING_LEN(path)) {

lib/rubygems.rb

@@ -9,7 +9,7 @@
 require 'thread'
 
 module Gem
-  VERSION = '2.4.5.4'
+  VERSION = '2.4.5.5'
 end
 
 # Must be first since it unloads the prelude from 1.9.2

lib/rubygems/package.rb

@@ -408,7 +408,7 @@ def install_location filename, destination_dir # :nodoc:
     destination = File.expand_path destination
 
     raise Gem::Package::PathError.new(destination, destination_dir) unless
-      destination.start_with? destination_dir
+      destination.start_with? destination_dir + '/'
 
     destination.untaint
     destination
@@ -587,6 +587,10 @@ def verify_files gem
       raise Gem::Package::FormatError.new \
               'package content (data.tar.gz) is missing', @gem
     end
+
+    if duplicates = @files.group_by {|f| f }.select {|k,v| v.size > 1 }.map(&:first) and duplicates.any?
+      raise Gem::Security::Exception, "duplicate files in the package: (#{duplicates.map(&:inspect).join(', ')})"
+    end
   end
 
   ##