Skip to content

Commit e2524e5

Browse files
Support :message_pack as a cache serializer format
This commit adds support for `:message_pack` as an option for `config.active_support.cache_format_version`. Cache entries written using the `6.1` or `7.0` formats can be read when using the `:message_pack` format. Additionally, cache entries written using the `:message_pack` format can now be read when using the `6.1` or `7.0` format. These behaviors makes it easy to migrate between formats without invalidating the entire cache.
1 parent dae8624 commit e2524e5

File tree

16 files changed

+378
-345
lines changed

16 files changed

+378
-345
lines changed

activesupport/CHANGELOG.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,16 @@
1+
* `config.active_support.cache_format_version` now accepts `:message_pack` as
2+
an option. `:message_pack` can reduce cache entry sizes and improve
3+
performance, but requires the [`msgpack` gem](https://rubygems.org/gems/msgpack)
4+
(>= 1.7.0).
5+
6+
Cache entries written using the `6.1` or `7.0` cache formats can be read
7+
when using the `:message_pack` cache format. Additionally, cache entries
8+
written using the `:message_pack` cache format can now be read when using
9+
the `6.1` or `7.0` cache formats. These behaviors makes it easy to migrate
10+
between formats without invalidating the entire cache.
11+
12+
*Jonathan Hefner*
13+
114
* `Object#deep_dup` no longer duplicate named classes and modules.
215

316
Before:

activesupport/lib/active_support/cache.rb

Lines changed: 9 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
require "active_support/core_ext/object/to_param"
99
require "active_support/core_ext/object/try"
1010
require "active_support/core_ext/string/inflections"
11+
require_relative "cache/serializer_with_fallback"
1112

1213
module ActiveSupport
1314
# See ActiveSupport::Cache::Store for documentation.
@@ -645,7 +646,14 @@ def clear(options = nil)
645646

646647
private
647648
def default_coder
648-
Coders[Cache.format_version]
649+
case Cache.format_version
650+
when 6.1
651+
Cache::SerializerWithFallback[:marshal_6_1]
652+
when 7.0
653+
Cache::SerializerWithFallback[:marshal_7_0]
654+
else
655+
Cache::SerializerWithFallback[Cache.format_version]
656+
end
649657
end
650658

651659
# Adds the namespace defined in the options to a pattern designed to
@@ -942,82 +950,6 @@ def load(payload)
942950
end
943951
end
944952

945-
module Coders # :nodoc:
946-
MARK_61 = "\x04\b".b.freeze # The one set by Marshal.
947-
MARK_70_UNCOMPRESSED = "\x00".b.freeze
948-
MARK_70_COMPRESSED = "\x01".b.freeze
949-
950-
class << self
951-
def [](version)
952-
case version
953-
when 6.1
954-
Rails61Coder
955-
when 7.0
956-
Rails70Coder
957-
else
958-
raise ArgumentError, "Unknown ActiveSupport::Cache.format_version: #{Cache.format_version.inspect}"
959-
end
960-
end
961-
end
962-
963-
module Loader
964-
extend self
965-
966-
def load(payload)
967-
if !payload.is_a?(String)
968-
ActiveSupport::Cache::Store.logger&.warn %{Payload wasn't a string, was #{payload.class.name} - couldn't unmarshal, so returning nil."}
969-
970-
return nil
971-
elsif payload.start_with?(MARK_70_UNCOMPRESSED)
972-
members = Marshal.load(payload.byteslice(1..-1))
973-
elsif payload.start_with?(MARK_70_COMPRESSED)
974-
members = Marshal.load(Zlib::Inflate.inflate(payload.byteslice(1..-1)))
975-
elsif payload.start_with?(MARK_61)
976-
return Marshal.load(payload)
977-
else
978-
ActiveSupport::Cache::Store.logger&.warn %{Invalid cache prefix: #{payload.byteslice(0).inspect}, expected "\\x00" or "\\x01"}
979-
980-
return nil
981-
end
982-
Entry.unpack(members)
983-
end
984-
end
985-
986-
module Rails61Coder
987-
include Loader
988-
extend self
989-
990-
def dump(entry)
991-
Marshal.dump(entry)
992-
end
993-
994-
def dump_compressed(entry, threshold)
995-
Marshal.dump(entry.compressed(threshold))
996-
end
997-
end
998-
999-
module Rails70Coder
1000-
include Loader
1001-
extend self
1002-
1003-
def dump(entry)
1004-
MARK_70_UNCOMPRESSED + Marshal.dump(entry.pack)
1005-
end
1006-
1007-
def dump_compressed(entry, threshold)
1008-
payload = Marshal.dump(entry.pack)
1009-
if payload.bytesize >= threshold
1010-
compressed_payload = Zlib::Deflate.deflate(payload)
1011-
if compressed_payload.bytesize < payload.bytesize
1012-
return MARK_70_COMPRESSED + compressed_payload
1013-
end
1014-
end
1015-
1016-
MARK_70_UNCOMPRESSED + payload
1017-
end
1018-
end
1019-
end
1020-
1021953
# This class is used to represent cache entries. Cache entries have a value, an optional
1022954
# expiration time, and an optional version. The expiration time is used to support the :race_condition_ttl option
1023955
# on the cache. The version is used to support the :version option on the cache for rejecting

activesupport/lib/active_support/cache/mem_cache_store.rb

Lines changed: 5 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -222,52 +222,12 @@ def stats
222222
end
223223

224224
private
225-
module Coders # :nodoc:
226-
class << self
227-
def [](version)
228-
case version
229-
when 6.1
230-
Rails61Coder
231-
when 7.0
232-
Rails70Coder
233-
else
234-
raise ArgumentError, "Unknown ActiveSupport::Cache.format_version #{Cache.format_version.inspect}"
235-
end
236-
end
237-
end
238-
239-
module Loader
240-
def load(payload)
241-
if payload.is_a?(Entry)
242-
payload
243-
else
244-
Cache::Coders::Loader.load(payload)
245-
end
246-
end
247-
end
248-
249-
module Rails61Coder
250-
include Loader
251-
extend self
252-
253-
def dump(entry)
254-
entry
255-
end
256-
257-
def dump_compressed(entry, threshold)
258-
entry.compressed(threshold)
259-
end
260-
end
261-
262-
module Rails70Coder
263-
include Cache::Coders::Rails70Coder
264-
include Loader
265-
extend self
266-
end
267-
end
268-
269225
def default_coder
270-
Coders[Cache.format_version]
226+
if Cache.format_version == 6.1
227+
Cache::SerializerWithFallback[:passthrough]
228+
else
229+
super
230+
end
271231
end
272232

273233
# Read an entry from the cache.
Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
# frozen_string_literal: true
2+
3+
module ActiveSupport
4+
module Cache
5+
module SerializerWithFallback # :nodoc:
6+
def self.[](format)
7+
if format.to_s.include?("message_pack") && !defined?(ActiveSupport::MessagePack)
8+
require "active_support/message_pack"
9+
end
10+
11+
SERIALIZERS.fetch(format)
12+
end
13+
14+
def dump_compressed(entry, threshold)
15+
dumped = dump(entry)
16+
try_compress(dumped, threshold) || dumped
17+
end
18+
19+
def load(dumped)
20+
if dumped.is_a?(String)
21+
dumped = decompress(dumped) if compressed?(dumped)
22+
23+
case
24+
when MessagePackWithFallback.dumped?(dumped)
25+
MessagePackWithFallback._load(dumped)
26+
when Marshal70WithFallback.dumped?(dumped)
27+
Marshal70WithFallback._load(dumped)
28+
when Marshal61WithFallback.dumped?(dumped)
29+
Marshal61WithFallback._load(dumped)
30+
else
31+
Cache::Store.logger&.warn("Unrecognized payload prefix #{dumped.byteslice(0).inspect}; deserializing as nil")
32+
nil
33+
end
34+
elsif PassthroughWithFallback.dumped?(dumped)
35+
PassthroughWithFallback._load(dumped)
36+
else
37+
Cache::Store.logger&.warn("Unrecognized payload class #{dumped.class}; deserializing as nil")
38+
nil
39+
end
40+
end
41+
42+
private
43+
ZLIB_HEADER = "\x78".b.freeze
44+
45+
def compressed?(dumped)
46+
dumped.start_with?(ZLIB_HEADER)
47+
end
48+
49+
def compress(dumped)
50+
Zlib::Deflate.deflate(dumped)
51+
end
52+
53+
def try_compress(dumped, threshold)
54+
if dumped.bytesize >= threshold
55+
compressed = compress(dumped)
56+
compressed unless compressed.bytesize >= dumped.bytesize
57+
end
58+
end
59+
60+
def decompress(compressed)
61+
Zlib::Inflate.inflate(compressed)
62+
end
63+
64+
module PassthroughWithFallback
65+
include SerializerWithFallback
66+
extend self
67+
68+
def dump(entry)
69+
entry
70+
end
71+
72+
def dump_compressed(entry, threshold)
73+
entry.compressed(threshold)
74+
end
75+
76+
def _load(entry)
77+
entry
78+
end
79+
80+
def dumped?(dumped)
81+
dumped.is_a?(Cache::Entry)
82+
end
83+
end
84+
85+
module Marshal61WithFallback
86+
include SerializerWithFallback
87+
extend self
88+
89+
MARSHAL_SIGNATURE = "\x04\x08".b.freeze
90+
91+
def dump(entry)
92+
Marshal.dump(entry)
93+
end
94+
95+
def dump_compressed(entry, threshold)
96+
Marshal.dump(entry.compressed(threshold))
97+
end
98+
99+
def _load(dumped)
100+
Marshal.load(dumped)
101+
end
102+
103+
def dumped?(dumped)
104+
dumped.start_with?(MARSHAL_SIGNATURE)
105+
end
106+
end
107+
108+
module Marshal70WithFallback
109+
include SerializerWithFallback
110+
extend self
111+
112+
MARK_UNCOMPRESSED = "\x00".b.freeze
113+
MARK_COMPRESSED = "\x01".b.freeze
114+
115+
def dump(entry)
116+
MARK_UNCOMPRESSED + Marshal.dump(entry.pack)
117+
end
118+
119+
def dump_compressed(entry, threshold)
120+
dumped = Marshal.dump(entry.pack)
121+
if compressed = try_compress(dumped, threshold)
122+
MARK_COMPRESSED + compressed
123+
else
124+
MARK_UNCOMPRESSED + dumped
125+
end
126+
end
127+
128+
def _load(marked)
129+
dumped = marked.byteslice(1..-1)
130+
dumped = decompress(dumped) if marked.start_with?(MARK_COMPRESSED)
131+
Cache::Entry.unpack(Marshal.load(dumped))
132+
end
133+
134+
def dumped?(dumped)
135+
dumped.start_with?(MARK_UNCOMPRESSED, MARK_COMPRESSED)
136+
end
137+
end
138+
139+
module MessagePackWithFallback
140+
include SerializerWithFallback
141+
extend self
142+
143+
def dump(entry)
144+
ActiveSupport::MessagePack::CacheSerializer.dump(entry.pack)
145+
end
146+
147+
def _load(dumped)
148+
packed = ActiveSupport::MessagePack::CacheSerializer.load(dumped)
149+
Cache::Entry.unpack(packed) if packed
150+
end
151+
152+
def dumped?(dumped)
153+
available? && ActiveSupport::MessagePack.signature?(dumped)
154+
end
155+
156+
private
157+
def available?
158+
return @available if defined?(@available)
159+
require "active_support/message_pack"
160+
@available = true
161+
rescue LoadError
162+
@available = false
163+
end
164+
end
165+
166+
SERIALIZERS = {
167+
passthrough: PassthroughWithFallback,
168+
marshal_6_1: Marshal61WithFallback,
169+
marshal_7_0: Marshal70WithFallback,
170+
message_pack: MessagePackWithFallback,
171+
}
172+
end
173+
end
174+
end

activesupport/lib/active_support/message_pack/cache_serializer.rb

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,34 +8,13 @@ module CacheSerializer
88
include Serializer
99
extend self
1010

11-
ZLIB_HEADER = "\x78"
12-
13-
def dump(entry)
14-
super(entry.pack)
15-
end
16-
17-
def dump_compressed(entry, threshold) # :nodoc:
18-
dumped = dump(entry)
19-
if dumped.bytesize >= threshold
20-
compressed = Zlib::Deflate.deflate(dumped)
21-
compressed.bytesize < dumped.bytesize ? compressed : dumped
22-
else
23-
dumped
24-
end
25-
end
26-
2711
def load(dumped)
28-
dumped = Zlib::Inflate.inflate(dumped) if compressed?(dumped)
29-
ActiveSupport::Cache::Entry.unpack(super)
12+
super
3013
rescue ActiveSupport::MessagePack::MissingClassError
3114
# Treat missing class as cache miss => return nil
3215
end
3316

3417
private
35-
def compressed?(dumped)
36-
dumped.start_with?(ZLIB_HEADER)
37-
end
38-
3918
def install_unregistered_type_handler
4019
Extensions.install_unregistered_type_fallback(message_pack_factory)
4120
end

0 commit comments

Comments
 (0)