New definition of whitespace & breakable whitespace

All whitespace can be removed by `strip` but only breakable whitespace
can be used as a delimiter by `split`, with updated docs for String.split/1

Signed-off-by: José Valim <[email protected]>

Author: amuino

lib/elixir/lib/string.ex

@@ -239,9 +239,11 @@ defmodule String do
   def printable?(<<>>), do: true
   def printable?(binary) when is_binary(binary), do: false
 
-  @doc """
+  @doc ~S"""
   Divides a string into substrings at each Unicode whitespace
-  occurrence with leading and trailing whitespace ignored.
+  occurrence with leading and trailing whitespace ignored. Groups
+  of whitespace are treated as a single occurrence. Divisions do
+  not occur on non-breaking whitespace.
 
   ## Examples
 
@@ -254,6 +256,9 @@ defmodule String do
       iex> String.split(" foo bar ")
       ["foo", "bar"]
 
+      iex> String.split("no\u00a0break")
+      ["no\u00a0break"]
+
   """
   @spec split(t) :: [t]
   defdelegate split(binary), to: String.Unicode

lib/elixir/test/elixir/string_test.exs

@@ -33,8 +33,12 @@ defmodule StringTest do
     assert String.split("foo bar ") == ["foo", "bar"]
     assert String.split(" foo bar ") == ["foo", "bar"]
     assert String.split("foo\t\n\v\f\r\sbar\n") == ["foo", "bar"]
-    assert String.split("foo" <> <<31>> <> "bar") == ["foo", "bar"]
     assert String.split("foo" <> <<194, 133>> <> "bar") == ["foo", "bar"]
+    # information separators are not considered whitespace
+    assert String.split("foo\u001Fbar") == ["foo\u001Fbar"]
+    # no-break space is excluded
+    assert String.split("foo\00A0bar") == ["foo\00A0bar"]
+    assert String.split("foo\u202Fbar") == ["foo\u202Fbar"]
 
     assert String.split("a,b,c", ",") == ["a", "b", "c"]
     assert String.split("a,b", ".") == ["a,b"]
@@ -179,14 +183,17 @@ defmodule StringTest do
     assert String.rstrip("   abc a") == "   abc a"
     assert String.rstrip("a  abc  a\n\n") == "a  abc  a"
     assert String.rstrip("a  abc  a\t\n\v\f\r\s") == "a  abc  a"
-    assert String.rstrip("a  abc  a " <> <<31>>) == "a  abc  a"
     assert String.rstrip("a  abc  a" <> <<194, 133>>) == "a  abc  a"
     assert String.rstrip("   abc aa", ?a) == "   abc "
     assert String.rstrip("   abc __", ?_) == "   abc "
     assert String.rstrip(" aaaaaaaaa", ?a) == " "
     assert String.rstrip("aaaaaaaaaa", ?a) == ""
     assert String.rstrip("]]]]]]]]]]", ?]) == ""
     assert String.rstrip("   cat 猫猫", ?猫) == "   cat "
+    # information separators are not whitespace
+    assert String.rstrip("a  abc  a \u001F") == "a  abc  a \u001F"
+    # no-break space
+    assert String.rstrip("a  abc  a \u00A0") == "a  abc  a"
   end
 
   test "lstrip" do
@@ -195,10 +202,13 @@ defmodule StringTest do
     assert String.lstrip("a  abc  a") == "a  abc  a"
     assert String.lstrip("\n\na  abc  a") == "a  abc  a"
     assert String.lstrip("\t\n\v\f\r\sa  abc  a") == "a  abc  a"
-    assert String.lstrip(<<31>> <> " a  abc  a") == "a  abc  a"
     assert String.lstrip(<<194, 133>> <> "a  abc  a") == "a  abc  a"
     assert String.lstrip("__  abc  _", ?_) == "  abc  _"
     assert String.lstrip("猫猫 cat   ", ?猫) == " cat   "
+    # information separators are not whitespace
+    assert String.lstrip("\u001F a  abc  a") == <<31>> <> " a  abc  a"
+    # no-break space
+    assert String.lstrip("\u00A0 a  abc  a") == "a  abc  a"
   end
 
   test "strip" do
@@ -208,6 +218,10 @@ defmodule StringTest do
     assert String.strip("a  abc  a\t\n\v\f\r\s") == "a  abc  a"
     assert String.strip("___  abc  ___", ?_) == "  abc  "
     assert String.strip("猫猫猫  cat  猫猫猫", ?猫) == "  cat  "
+    # no-break space
+    assert String.strip("\u00A0a  abc  a\u00A0") == "a  abc  a"
+    # whitespace defined as a range
+    assert String.strip("\u2008a  abc  a\u2005") == "a  abc  a"
   end
 
   test "rjust" do

lib/elixir/unicode/WhiteSpace.txt

@@ -0,0 +1,11 @@
+0009..000D    ; White_Space # Cc   [5] <control-0009>..<control-000D>
+0020          ; White_Space # Zs       SPACE
+0085          ; White_Space # Cc       <control-0085>
+00A0          ; White_Space # Zs       NO-BREAK SPACE
+1680          ; White_Space # Zs       OGHAM SPACE MARK
+2000..200A    ; White_Space # Zs  [11] EN QUAD..HAIR SPACE
+2028          ; White_Space # Zl       LINE SEPARATOR
+2029          ; White_Space # Zp       PARAGRAPH SEPARATOR
+202F          ; White_Space # Zs       NARROW NO-BREAK SPACE
+205F          ; White_Space # Zs       MEDIUM MATHEMATICAL SPACE
+3000          ; White_Space # Zs       IDEOGRAPHIC SPACE

lib/elixir/unicode/unicode.ex

@@ -12,11 +12,26 @@ defmodule String.Unicode do
   @moduledoc false
   def version, do: {7, 0, 0}
 
+  # WhiteSpace.txt is extracted from Unicode's PropList.txt (just the White_Space property)
+  prop_path = Path.join(__DIR__, "WhiteSpace.txt")
+  prop_range_line_regex = ~r/\A([0-9A-F]{4})(?:\.\.([0-9A-F]{4}))?/
+  whitespace = Enum.reduce File.stream!(prop_path), [], fn(line, acc) ->
+    case Regex.run(prop_range_line_regex, line, capture: :all_but_first) do
+      [single] ->
+        [to_binary.(single) | acc]
+      [first, last] ->
+        range = String.to_integer(first, 16)..String.to_integer(last, 16)
+        acc ++ Enum.map(range, &to_binary.(Integer.to_string(&1, 16)))
+      _ ->
+        acc
+    end
+  end
+
   data_path = Path.join(__DIR__, "UnicodeData.txt")
 
-  {codes, whitespace} = Enum.reduce File.stream!(data_path), {[], []}, fn(line, {cacc, wacc}) ->
+  {codes, breakable_whitespace} = Enum.reduce File.stream!(data_path), {[], whitespace}, fn(line, {cacc, wacc}) ->
     [codepoint, _name, _category,
-     _class, bidi, _decomposition,
+     _class, _bidi, decomposition,
      _numeric_1, _numeric_2, _numeric_3,
      _bidi_mirror, _unicode_1, _iso,
      upper, lower, title] = :binary.split(line, ";", [:global])
@@ -30,8 +45,8 @@ defmodule String.Unicode do
            to_binary.(lower),
            to_binary.(title)} | cacc],
          wacc}
-      bidi in ["B", "S", "WS"] ->
-        {cacc, [to_binary.(codepoint) | wacc]}
+      String.starts_with?(decomposition, "<noBreak>") ->
+        {cacc, List.delete(wacc, to_binary.(codepoint))}
       true ->
         {cacc, wacc}
     end
@@ -156,7 +171,7 @@ defmodule String.Unicode do
     :lists.reverse do_split(string, "", [])
   end
 
-  for codepoint <- whitespace do
+  for codepoint <- breakable_whitespace do
     defp do_split(unquote(codepoint) <> rest, buffer, acc) do
       do_split(rest, "", add_buffer_to_acc(buffer, acc))
     end