Skip to content

Commit 213c50f

Browse files
bobzhangbobzhang
committed
reduce the size of cmi/cmj
1 parent c74139c commit 213c50f

25 files changed

+1459
-1594
lines changed

‎jscomp/core/builtin_cmi_datasets.ml

Lines changed: 142 additions & 224 deletions
Large diffs are not rendered by default.

‎jscomp/core/builtin_cmi_datasets.mli

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
3+
4+
(* TODO: we can hide some
5+
more internal units
6+
*)
7+
val module_sets : string array
8+
val module_sets_cmi : Cmi_format.cmi_infos Lazy.t array

‎jscomp/core/builtin_cmj_datasets.ml

Lines changed: 143 additions & 223 deletions
Large diffs are not rendered by default.

‎jscomp/core/builtin_cmj_datasets.mli

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
val module_sets : string array
2+
val module_sets_cmj : Js_cmj_format.t Lazy.t array
3+
val query_by_name : string -> Js_cmj_format.t option

‎jscomp/core/js_cmj_format.ml

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -44,36 +44,34 @@ type effect = string option
4444
let single_na = Single Lam_arity.na
4545
(** we don't force people to use package *)
4646
type cmj_case = Ext_namespace.file_kind
47-
47+
48+
type keyed_cmj_value = { name : string ; arity : arity ; persistent_closed_lambda : Lam.t option}
4849
type keyed_cmj_values
49-
= (string * cmj_value) array
50+
= keyed_cmj_value array
5051

5152
type t = {
5253
values : keyed_cmj_values ;
5354
pure : bool;
5455
npm_package_path : Js_packages_info.t ;
5556
cmj_case : cmj_case;
5657
}
57-
(* let empty_values = [||] *)
58-
let mk ~values ~effect ~npm_package_path ~cmj_case : t =
58+
59+
let mk ~(values:cmj_value Map_string.t) ~effect ~npm_package_path ~cmj_case : t =
5960
{
60-
values = Map_string.to_sorted_array values;
61+
values = Map_string.to_sorted_array_with_f values (fun k v -> {
62+
name = k ;
63+
arity = v.arity;
64+
persistent_closed_lambda = v.persistent_closed_lambda
65+
});
6166
pure = effect = None ;
6267
npm_package_path;
6368
cmj_case
6469
}
6570

66-
let cmj_magic_number = "BUCKLE20171012"
67-
let cmj_magic_number_length =
68-
String.length cmj_magic_number
69-
70-
71-
72-
let digest_length = 16 (*16 chars *)
7371

7472
let verify_magic_in_beg ic =
75-
let buffer = really_input_string ic cmj_magic_number_length in
76-
if buffer <> cmj_magic_number then
73+
let buffer = really_input_string ic Ext_cmj_magic.cmj_magic_number_length in
74+
if buffer <> Ext_cmj_magic.cmj_magic_number then
7775
Ext_fmt.failwithf ~loc:__LOC__
7876
"cmj files have incompatible versions, please rebuilt using the new compiler : %s"
7977
__LOC__
@@ -98,21 +96,20 @@ let from_file_with_digest name : t * Digest.t =
9896

9997

10098
let from_string s : t =
101-
let magic_number = String.sub s 0 cmj_magic_number_length in
102-
if magic_number = cmj_magic_number then
103-
Marshal.from_string s (digest_length + cmj_magic_number_length)
99+
let magic_number = String.sub s 0 Ext_cmj_magic.cmj_magic_number_length in
100+
if magic_number = Ext_cmj_magic.cmj_magic_number then
101+
Marshal.from_string s Ext_cmj_magic.header_length
104102
else
105103
Ext_fmt.failwithf ~loc:__LOC__
106104
"cmj files have incompatible versions, please rebuilt using the new compiler : %s"
107105
__LOC__
108106

109-
let fixed_length = cmj_magic_number_length + digest_length
110107

111108
let for_sure_not_changed (name : string) (header : string) =
112109
if Sys.file_exists name then
113110
let ic = open_in_bin name in
114111
let holder =
115-
really_input_string ic fixed_length in
112+
really_input_string ic Ext_cmj_magic.header_length in
116113
close_in ic;
117114
holder = header
118115
else false
@@ -123,21 +120,21 @@ let for_sure_not_changed (name : string) (header : string) =
123120
let to_file name ~check_exists (v : t) =
124121
let s = Marshal.to_string v [] in
125122
let cur_digest = Digest.string s in
126-
let header = cmj_magic_number ^ cur_digest in
123+
let header = Ext_cmj_magic.cmj_magic_number ^ cur_digest in
127124
if not (check_exists && for_sure_not_changed name header) then
128125
let oc = open_out_bin name in
129126
output_string oc header;
130127
output_string oc s;
131128
close_out oc
132129

133-
let keyComp (a : string) (b,_) =
134-
Map_string.compare_key a b
130+
let keyComp (a : string) b =
131+
Map_string.compare_key a b.name
135132

136133
let not_found = single_na, None
137-
let get_result midVal =
138-
let (_,cmj_value) = midVal in
139-
cmj_value.arity,
140-
if Js_config.get_cross_module_inline () then cmj_value.persistent_closed_lambda
134+
135+
let get_result midVal =
136+
midVal.arity,
137+
if Js_config.get_cross_module_inline () then midVal.persistent_closed_lambda
141138
else None
142139

143140
let rec binarySearchAux arr lo hi (key : string) =
@@ -149,13 +146,13 @@ let rec binarySearchAux arr lo hi (key : string) =
149146
else if c < 0 then (* a[lo] =< key < a[mid] <= a[hi] *)
150147
if hi = mid then
151148
let loVal = (Array.unsafe_get arr lo) in
152-
if fst loVal = key then get_result loVal
149+
if loVal.name = key then get_result loVal
153150
else not_found
154151
else binarySearchAux arr lo mid key
155152
else (* a[lo] =< a[mid] < key <= a[hi] *)
156153
if lo = mid then
157154
let hiVal = (Array.unsafe_get arr hi) in
158-
if fst hiVal = key then get_result hiVal
155+
if hiVal.name = key then get_result hiVal
159156
else not_found
160157
else binarySearchAux arr mid hi key
161158

@@ -215,7 +212,7 @@ let pp_cmj
215212
f "effect: %s\n"
216213
(if pure then "pure" else "not pure");
217214
Ext_array.iter values
218-
(fun (k , {arity; persistent_closed_lambda}) ->
215+
(fun ({name = k ; arity; persistent_closed_lambda}) ->
219216
match arity with
220217
| Single arity ->
221218
f "%s: %s\n" k (Format.asprintf "%a" Lam_arity.print arity);

‎jscomp/ext/ext_cmj_magic.ml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
(* Copyright (C) 2020 Authors of BuckleScript
2+
*
3+
* This program is free software: you can redistribute it and/or modify
4+
* it under the terms of the GNU Lesser General Public License as published by
5+
* the Free Software Foundation, either version 3 of the License, or
6+
* (at your option) any later version.
7+
*
8+
* In addition to the permissions granted to you by the LGPL, you may combine
9+
* or link a "work that uses the Library" with a publicly distributed version
10+
* of this file to produce a combined library or application, then distribute
11+
* that combined work under the terms of your choosing, with no requirement
12+
* to comply with the obligations normally placed on you by section 4 of the
13+
* LGPL version 3 (or the corresponding section of a later version of the LGPL
14+
* should you choose to use a later version).
15+
*
16+
* This program is distributed in the hope that it will be useful,
17+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
18+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+
* GNU Lesser General Public License for more details.
20+
*
21+
* You should have received a copy of the GNU Lesser General Public License
22+
* along with this program; if not, write to the Free Software
23+
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *)
24+
25+
26+
let cmj_magic_number = "BUCKLE20200209"
27+
let cmj_magic_number_length =
28+
String.length cmj_magic_number
29+
30+
31+
32+
let digest_length = 16 (*16 chars *)
33+
34+
let header_length = cmj_magic_number_length + digest_length

‎jscomp/ext/map.cppo.ml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ let singleton = Map_gen.singleton
3232
let cardinal = Map_gen.cardinal
3333
let bindings = Map_gen.bindings
3434
let to_sorted_array = Map_gen.to_sorted_array
35+
let to_sorted_array_with_f = Map_gen.to_sorted_array_with_f
3536
let keys = Map_gen.keys
3637
let choose = Map_gen.choose
3738
let partition = Map_gen.partition

‎jscomp/ext/map_gen.ml

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,14 @@ let rec bindings_aux accu = function
3434
let bindings s =
3535
bindings_aux [] s
3636

37-
37+
let rec fill_array_with_f (s : _ t) i arr f : int =
38+
match s with
39+
| Empty -> i
40+
| Node ( l ,k,v,r,_) ->
41+
let inext = fill_array_with_f l i arr f in
42+
Array.unsafe_set arr inext (f k v);
43+
fill_array_with_f r (inext + 1) arr f
44+
3845
let rec fill_array_aux (s : _ t) i arr : int =
3946
match s with
4047
| Empty -> i
@@ -43,6 +50,7 @@ let rec fill_array_aux (s : _ t) i arr : int =
4350
Array.unsafe_set arr inext (k,v);
4451
fill_array_aux r (inext + 1) arr
4552

53+
4654
let to_sorted_array (s : ('key,'a) t) : ('key * 'a ) array =
4755
match s with
4856
| Empty -> [||]
@@ -53,6 +61,18 @@ let to_sorted_array (s : ('key,'a) t) : ('key * 'a ) array =
5361
Array.make len (k,v) in
5462
ignore (fill_array_aux s 0 arr : int);
5563
arr
64+
65+
let to_sorted_array_with_f (type key a b ) (s : (key,a) t) (f : key -> a -> b): b array =
66+
match s with
67+
| Empty -> [||]
68+
| Node(l,k,v,r,_) ->
69+
let len =
70+
cardinal_aux (cardinal_aux 1 r) l in
71+
let arr =
72+
Array.make len (f k v) in
73+
ignore (fill_array_with_f s 0 arr f: int);
74+
arr
75+
5676
let rec keys_aux accu = function
5777
Empty -> accu
5878
| Node(l, v, _, r, _) -> keys_aux (v :: keys_aux accu r) l
@@ -279,6 +299,8 @@ module type S =
279299
val mem: 'a t -> key -> bool
280300
val to_sorted_array :
281301
'a t -> (key * 'a ) array
302+
val to_sorted_array_with_f :
303+
'a t -> (key -> 'a -> 'b) -> 'b array
282304
val add: 'a t -> key -> 'a -> 'a t
283305
(** [add x y m]
284306
If [x] was already bound in [m], its previous binding disappears. *)

‎jscomp/main/cmij_main.ml

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ let from_cmj (files : string list) (output_file : string) =
4646
let keys = Ext_list.map files (fun x -> "\"" ^Filename.basename x ^ "\"") in
4747
let v = open_out_bin output_file in
4848
Ext_pervasives.finally v ~clean:close_out (fun f ->
49+
output_string f {|
50+
let i s = lazy (Marshal.from_string s 0)
51+
|};
4952
output_string f
5053
(Printf.sprintf {|let module_sets = [|
5154
%s
@@ -58,8 +61,10 @@ let from_cmj (files : string list) (output_file : string) =
5861
%s
5962
|]
6063
|} (String.concat ";\n" (Ext_list.map files (fun file ->
61-
Printf.sprintf "lazy (Js_cmj_format.from_string %S )"
62-
(Ext_io.load_file file)))));
64+
Printf.sprintf "i %S"
65+
(let content = Ext_io.load_file file in
66+
String.sub content Ext_cmj_magic.header_length (String.length content - Ext_cmj_magic.header_length)
67+
)))));
6368
output_string f "\n";
6469
output_string f {|
6570
let query_by_name s =
@@ -81,6 +86,9 @@ let from_cmi (files : string list) (output_file : string) =
8186
let keys = Ext_list.map files (fun x -> "\"" ^ Ext_filename.module_name x ^ "\"") in
8287
let v = open_out_bin output_file in
8388
Ext_pervasives.finally v ~clean:close_out (fun f ->
89+
output_string f {|
90+
let i s = lazy (Marshal.from_string s 0)
91+
|};
8492
output_string f
8593
(Printf.sprintf {|let module_sets = [|
8694
%s
@@ -93,7 +101,7 @@ let from_cmi (files : string list) (output_file : string) =
93101
%s
94102
|]
95103
|} (String.concat ";\n" (Ext_list.map files (fun file ->
96-
Printf.sprintf "lazy (Marshal.from_string %S 0)"
104+
Printf.sprintf "i %S"
97105
(let content = (Cmi_format.read_cmi file) in
98106
Marshal.to_string content []
99107
(* let header_len = (String.length Config.cmi_magic_number) in
@@ -107,12 +115,14 @@ let from_cmi (files : string list) (output_file : string) =
107115
let stdlib = "stdlib-406"
108116

109117
let () =
110-
from_cmj ( Ext_list.append (get_files Literals.suffix_cmj stdlib)
111-
(Ext_list.append (get_files Literals.suffix_cmj "runtime")
112-
(get_files Literals.suffix_cmj "others")))
118+
from_cmj (
119+
Filename.concat "runtime" "js.cmj" ::
120+
get_files Literals.suffix_cmj stdlib @
121+
get_files Literals.suffix_cmj "others")
113122
(Filename.concat "core" "builtin_cmj_datasets.ml");
114-
from_cmi ( Ext_list.append (get_files Literals.suffix_cmi stdlib)
115-
(Ext_list.append (get_files Literals.suffix_cmi "runtime")
116-
(get_files Literals.suffix_cmi "others")))
123+
from_cmi (
124+
Filename.concat "runtime" "js.cmi" ::
125+
get_files Literals.suffix_cmi stdlib @
126+
get_files Literals.suffix_cmi "others")
117127
(Filename.concat "core" "builtin_cmi_datasets.ml")
118128

‎jscomp/others/release.ninja

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ build others/js_exn.cmi : cc others/js_exn.mli | runtime
2626
build others/js_float.cmi others/js_float.cmj : cc others/js_float.ml | runtime
2727
build others/js_global.cmi others/js_global.cmj : cc others/js_global.ml | runtime
2828
build others/js_int.cmi others/js_int.cmj : cc others/js_int.ml | runtime
29+
build others/js_internalOO.cmi others/js_internalOO.cmj : cc others/js_internalOO.ml | runtime
2930
build others/js_json.cmj : cc_cmi others/js_json.ml | others/js_array2.cmj others/js_dict.cmj others/js_json.cmi others/js_string.cmj others/js_types.cmj runtime
3031
build others/js_json.cmi : cc others/js_json.mli | others/js_dict.cmi others/js_null.cmi others/js_string.cmj others/js_types.cmi runtime
3132
build others/js_list.cmj : cc_cmi others/js_list.ml | others/js_array2.cmj others/js_list.cmi others/js_vector.cmj runtime
@@ -54,7 +55,7 @@ build others/js_undefined.cmj : cc_cmi others/js_undefined.ml | others/js_exn.cm
5455
build others/js_undefined.cmi : cc others/js_undefined.mli | runtime
5556
build others/js_vector.cmj : cc_cmi others/js_vector.ml | others/js_array2.cmj others/js_vector.cmi runtime
5657
build others/js_vector.cmi : cc others/js_vector.mli | runtime
57-
build js_pkg : phony others/js_array.cmi others/js_array.cmj others/js_array2.cmi others/js_array2.cmj others/js_cast.cmi others/js_cast.cmj others/js_console.cmi others/js_console.cmj others/js_date.cmi others/js_date.cmj others/js_dict.cmi others/js_dict.cmj others/js_exn.cmi others/js_exn.cmj others/js_float.cmi others/js_float.cmj others/js_global.cmi others/js_global.cmj others/js_int.cmi others/js_int.cmj others/js_json.cmi others/js_json.cmj others/js_list.cmi others/js_list.cmj others/js_mapperRt.cmi others/js_mapperRt.cmj others/js_math.cmi others/js_math.cmj others/js_null.cmi others/js_null.cmj others/js_null_undefined.cmi others/js_null_undefined.cmj others/js_obj.cmi others/js_obj.cmj others/js_option.cmi others/js_option.cmj others/js_promise.cmi others/js_promise.cmj others/js_re.cmi others/js_re.cmj others/js_result.cmi others/js_result.cmj others/js_string.cmi others/js_string.cmj others/js_string2.cmi others/js_string2.cmj others/js_typed_array.cmi others/js_typed_array.cmj others/js_typed_array2.cmi others/js_typed_array2.cmj others/js_types.cmi others/js_types.cmj others/js_undefined.cmi others/js_undefined.cmj others/js_vector.cmi others/js_vector.cmj
58+
build js_pkg : phony others/js_array.cmi others/js_array.cmj others/js_array2.cmi others/js_array2.cmj others/js_cast.cmi others/js_cast.cmj others/js_console.cmi others/js_console.cmj others/js_date.cmi others/js_date.cmj others/js_dict.cmi others/js_dict.cmj others/js_exn.cmi others/js_exn.cmj others/js_float.cmi others/js_float.cmj others/js_global.cmi others/js_global.cmj others/js_int.cmi others/js_int.cmj others/js_internalOO.cmi others/js_internalOO.cmj others/js_json.cmi others/js_json.cmj others/js_list.cmi others/js_list.cmj others/js_mapperRt.cmi others/js_mapperRt.cmj others/js_math.cmi others/js_math.cmj others/js_null.cmi others/js_null.cmj others/js_null_undefined.cmi others/js_null_undefined.cmj others/js_obj.cmi others/js_obj.cmj others/js_option.cmi others/js_option.cmj others/js_promise.cmi others/js_promise.cmj others/js_re.cmi others/js_re.cmj others/js_result.cmi others/js_result.cmj others/js_string.cmi others/js_string.cmj others/js_string2.cmi others/js_string2.cmj others/js_typed_array.cmi others/js_typed_array.cmj others/js_typed_array2.cmi others/js_typed_array2.cmj others/js_types.cmi others/js_types.cmj others/js_undefined.cmi others/js_undefined.cmj others/js_vector.cmi others/js_vector.cmj
5859
build others/belt_Array.cmj : cc_cmi others/belt_Array.ml | others/belt.cmi others/belt_Array.cmi others/js_math.cmj js_pkg runtime
5960
build others/belt_Array.cmi : cc others/belt_Array.mli | runtime
6061
build others/belt_Debug.cmj : cc_cmi others/belt_Debug.ml | others/belt.cmi others/belt_Debug.cmi js_pkg runtime

0 commit comments

Comments
 (0)