diff --git a/rebar.config b/rebar.config index cef4c325..0941f30a 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,8 @@ {port_specs, [{"priv/bitcask.so", ["c_src/*.c"]}]}. {deps, [ - {meck, "0.8.1", {git, "git://github.com/basho/meck.git", {tag, "0.8.1"}}} + {meck, "0.8.1", {git, "git://github.com/basho/meck.git", {tag, "0.8.1"}}}, + {snappy, ".*", {git, "git://github.com/basho/snappy-erlang-nif.git", "master"}} ]}. {port_env, [ diff --git a/src/bitcask.erl b/src/bitcask.erl index 4243e315..165fe661 100644 --- a/src/bitcask.erl +++ b/src/bitcask.erl @@ -217,7 +217,8 @@ get(Ref, Key, TryNum) -> {ok, _Key, ?TOMBSTONE} -> not_found; {ok, _Key, Value} -> - {ok, Value}; + Val = maybe_decompress(Value), + {ok, Val}; {error, eof} -> not_found; {error, _} = Err -> @@ -1222,9 +1223,11 @@ do_put(Key, Value, #bc_state{write_file = WriteFile} = State, Retries, _LastErr) end, Tstamp = bitcask_time:tstamp(), + + CValue = maybe_compress(Value, State#bc_state.opts), {ok, WriteFile2, Offset, Size} = bitcask_fileops:write( State2#bc_state.write_file, - Key, Value, Tstamp), + Key, CValue, Tstamp), case bitcask_nifs:keydir_put(State2#bc_state.keydir, Key, bitcask_fileops:file_tstamp(WriteFile2), Size, Offset, Tstamp, true) of @@ -1333,6 +1336,68 @@ expiry_merge([File | Files], LiveKeyDir, Acc0) -> end, expiry_merge(Files, LiveKeyDir, Acc). +enable_compression(Opts) -> + case get_opt(enable_compression, Opts) of + true -> + true; + _ -> + false + end. + +compression_ratio_threshold(Opts) -> + case get_opt(compression_ratio_threshold, Opts) of + T when is_float(T) -> + T; + _ -> + 1.0 + end. + +compression_size_threshold(Opts) -> + case get_opt(compression_size_threshold, Opts) of + T when is_integer(T) -> + T; + _ -> + 1024 + end. + +check_compression_threshold(Value, Compressed, Opts) -> + ValueSize = byte_size(Value), + SizeThreshold = compression_size_threshold(Opts), + case ValueSize > SizeThreshold of + true -> + RatioThreshold = compression_ratio_threshold(Opts), + case (byte_size(Compressed) / byte_size(Value)) of + Ratio when Ratio < RatioThreshold -> + Compressed; + _ -> + Value + end; + _ -> + Value + end. + +maybe_compress(Value, Opts) -> + case enable_compression(Opts) of + true -> + case snappy:compress(Value) of + {ok, Compressed} -> + check_compression_threshold(Value, Compressed, Opts); + _ -> + Value + end; + false -> + Value + end. + +maybe_decompress(Value) -> + case snappy:is_valid(Value) of + true -> + {ok, Val} = snappy:decompress(Value), + Val; + _ -> + Value + end. + %% =================================================================== %% EUnit tests %% ===================================================================