Support 3.4-style symbols as hash keys

Author: janlelis

CHANGELOG.md

@@ -3,7 +3,7 @@
 ## 3.0 (unreleased)
 * Require Ruby 3
 * Relax Ruby version requirement to allow Ruby 4.0
-* Support Ruby 3.4 hash inspects (TODO)
+* Support Ruby 3.4 hash inspects
 * Support Ruby 4.0 changes (TODO)
 
 ## 2.2.2

lib/wirb/tokenizer.rb

@@ -160,18 +160,18 @@ def self.run(str)
           end
 
         when :class
-          case c
-          when /[a-z0-9_]/i
+          if c =~ /[a-z0-9_]/i
             @token << c
-          else
-            if @token =~ /^(Infinity|NaN)$/
-              set_state[:special_number, :repeat]
-            elsif c ==':' && nc == ':'
-              pass_state[]
-              pass[:class_separator, '::']
-            elsif !(c == ':' && lc == ':')
-              pass_state[:remove, :repeat]
-            end
+          elsif c == ":" && get_previous_state[:hash]
+            pass_custom_state[:symbol, :remove, :repeat]
+            @refers_seen[-1] = true
+          elsif @token =~ /^(Infinity|NaN)$/
+            set_state[:special_number, :repeat]
+          elsif c ==':' && nc == ':'
+            pass_state[]
+            pass[:class_separator, '::']
+          elsif !(c == ':' && lc == ':')
+            pass_state[:remove, :repeat]
           end
 
         when :symbol
@@ -208,10 +208,15 @@ def self.run(str)
           end
 
         when :string
-          if c == '"' && ( !( @token =~ /\\+$/; $& ) || $&.size % 2 == 0 ) # allow escaping of " and
-            pass[:open_string, '"']                              # work around \\
-            pass_state[:remove]
-            pass[:close_string, '"']
+          if c == '"' && ( !( @token =~ /\\+$/; $& ) || $&.size % 2 == 0 ) # allow escaping of " and work around \\
+            if nc == ':' && get_previous_state[:hash]
+              set_state[:symbol_string, :repeat]
+              @refers_seen[-1] = true
+            else
+              pass[:open_string, '"']
+              pass_state[:remove]
+              pass[:close_string, '"']
+            end
           else
             @token << c
           end
@@ -237,6 +242,9 @@ def self.run(str)
           if c =~ /[a-z0-9_]/i
             @token << c
             pass_custom_state[@token.to_sym, :remove] if %w[nil false true].include?(@token)
+          elsif c == ":"
+            pass_custom_state[:symbol, :remove, :repeat]
+            @refers_seen[-1] = true
           else
             pass_state[:remove, :repeat]
           end

spec/tokenizer_hash_spec.rb

@@ -67,11 +67,71 @@
     end
   end
 
-  please do check({:hallo => {1=>Set[2,3,4]}})
+  please do check({:Symbol => :value})
     if symbol_hash_keys?
-      fail
       tokens.should == [
-        # TODO
+        [:open_hash, "{"],
+        [:symbol, "Symbol"],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:symbol_prefix, ":"],
+        [:symbol, "value"],
+        [:close_hash, "}"],
+      ]
+    else
+      tokens.should == [
+        [:open_hash, "{"],
+        [:symbol_prefix, ":"],
+        [:symbol, "Symbol"],
+        [:refers, "=>"],
+        [:symbol_prefix, ":"],
+        [:symbol, "value"],
+        [:close_hash, "}"],
+      ]
+    end
+  end
+
+  please do check({Module => :value})
+    tokens.should == [
+      [:open_hash, "{"],
+      [:class, "Module"],
+      ([:whitespace, " "] if spaced_hashes?),
+      [:refers, "=>"],
+      ([:whitespace, " "] if spaced_hashes?),
+      [:symbol_prefix, ":"],
+      [:symbol, "value"],
+      [:close_hash, "}"],
+    ].compact
+  end
+
+  please do check({:hallo => {1=>Set[2,3,4]}})
+    if symbol_hash_keys?
+      tokens.should be_like [
+        [:open_hash, "{"],
+        [:symbol, "hallo"],
+        [:symbol_prefix, ":"],
+        ([:whitespace, " "] if spaced_hashes?),
+        [:open_hash, "{"],
+        [:number, "1"],
+        ([:whitespace, " "] if spaced_hashes?),
+        [:refers, "=>"],
+        ([:whitespace, " "] if spaced_hashes?),
+        [:open_object, "#<"],
+        [:object_class, "Set"],
+        [:object_description_prefix, ":"],
+        [:whitespace, " "],
+        [:open_set, "{"],
+        [:number, /\d+/],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, /\d+/],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:number, /\d+/],
+        [:close_set, "}"],
+        [:close_object, ">"],
+        [:close_hash, "}"],
+        [:close_hash, "}"],
       ]
     else
       tokens.should be_like [

spec/tokenizer_symbol_spec.rb

@@ -99,7 +99,67 @@
 
   please do check [[:hallo],{:'hallo'=>:'test'}, {:'hallo='=>:'test='}, {:'<'=>9}, {:'<=>'=>9}, {:'<'=>:'<=>'}]
     if symbol_hash_keys?
-      fail
+      tokens.should == [
+        [:open_array, "["],
+        [:open_array, "["],
+        [:symbol_prefix, ":"],
+        [:symbol, "hallo"],
+        [:close_array, "]"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_hash, "{"],
+        [:symbol, "hallo"],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:symbol_prefix, ":"],
+        [:symbol, "test"],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_hash, "{"],
+        [:open_symbol_string, "\""],
+        [:symbol_string, "hallo="],
+        [:close_symbol_string, "\""],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:symbol_prefix, ":"],
+        [:symbol, "test="],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_hash, "{"],
+        [:open_symbol_string, "\""],
+        [:symbol_string, "<"],
+        [:close_symbol_string, "\""],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:number, "9"],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_hash, "{"],
+        [:open_symbol_string, "\""],
+        [:symbol_string, "<=>"],
+        [:close_symbol_string, "\""],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:number, "9"],
+        [:close_hash, "}"],
+        [:comma, ","],
+        [:whitespace, " "],
+        [:open_hash, "{"],
+        [:open_symbol_string, "\""],
+        [:symbol_string, "<"],
+        [:close_symbol_string, "\""],
+        [:symbol_prefix, ":"],
+        [:whitespace, " "],
+        [:symbol_prefix, ":"],
+        # [:symbol, "<=>"], # MAYBE fixme
+        [:symbol, "<"],
+        [:symbol, "=>"],
+        [:close_hash, "}"],
+        [:close_array, "]"],
+      ]
     else
       tokens.should == [
         [:open_array, "["],