Skip to content

Commit 7c1a888

Browse files
bkaminsbkamins
authored
precompilation for 1.4 release (#3182)
1 parent 1615476 commit 7c1a888

File tree

5 files changed

+38
-116
lines changed

5 files changed

+38
-116
lines changed

‎Project.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
1717
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
1818
REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
1919
Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
20+
SnoopPrecompile = "66db9d55-30c0-4569-8b51-7e840670fc0c"
2021
SortingAlgorithms = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
2122
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
2223
TableTraits = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
@@ -34,6 +35,7 @@ PooledArrays = "1.4.2"
3435
PrettyTables = "2.1"
3536
Reexport = "0.1, 0.2, 1"
3637
ShiftedArrays = "1, 2"
38+
SnoopPrecompile = "1"
3739
SortingAlgorithms = "0.1, 0.2, 0.3, 1"
3840
TableTraits = "0.4, 1"
3941
Tables = "1.9.0"

‎src/DataFrames.jl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ using Markdown
1010
using PrettyTables
1111
using Random
1212
using Tables: ByRow
13+
import SnoopPrecompile
1314

1415
import DataAPI,
1516
DataAPI.allcombinations,
@@ -173,6 +174,5 @@ include("other/metadata.jl")
173174
include("deprecated.jl")
174175

175176
include("other/precompile.jl")
176-
precompile()
177177

178178
end # module DataFrames

‎src/other/precompile.jl‎

Lines changed: 35 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,37 @@
1-
# precompile.jl contains precompilation directives for all methods whose compilation
2-
# is triggered by running DataFrames tests and takes more than a given threshold
3-
# of 0.1.
4-
# `precompile` calls are generated using SnoopCompile via:
5-
# using SnoopCompileCore
6-
# inf_timing = @snoopi tmin=0.1 include("test/runtests.jl")
7-
# using SnoopCompile
8-
# pc = SnoopCompile.parcel(inf_timing)
9-
# SnoopCompile.write("src/other/precompile_tmp.jl", pc[:DataFrames], always=true)
10-
# and then hand edited. The editing steps are:
11-
# * removing signatures with anonymous functions
12-
# * removing signatures with specific NamedTuples passed as arguments except `x1` as name or kwargs
13-
# (which is generated internally)
14-
# * changing Int64 to Int (to handle 32-bit architectures correctly)
15-
# * disabling precompilation on Julia older than 1.5
16-
# * run @warnpcfail check for all=true and all=false both on Julia stable and nightly
1+
import SnoopPrecompile
172

18-
function precompile(all=false)
19-
all || ccall(:jl_generating_output, Cint, ()) == 1 || return nothing
20-
21-
Base.precompile(Tuple{Aggregate{typeof(std), Nothing},Vector{Union{Missing, Int}},GroupedDataFrame{DataFrame}})
22-
Base.precompile(Tuple{Core.kwftype(typeof(describe)),NamedTuple{(:cols,), Tuple{Vector{Pair{Symbol, Pair{Function, Symbol}}}}},typeof(describe),DataFrame})
23-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Symbol, Bool, Pair{Bool, Bool}, Pair{String, String}}},typeof(innerjoin),DataFrame,DataFrame})
24-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Symbol, Bool, Pair{Bool, Bool}, Pair{String, Symbol}}},typeof(innerjoin),DataFrame,DataFrame})
25-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Symbol, Bool, Pair{Bool, Bool}, Pair{Symbol, Symbol}}},typeof(innerjoin),DataFrame,DataFrame})
26-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Vector{Any}, Bool, Pair{Bool, Bool}, Pair{String, Symbol}}},typeof(innerjoin),DataFrame,DataFrame})
27-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on, :renamecols), Tuple{Vector{Any}, Pair{String, String}}},typeof(innerjoin),DataFrame,DataFrame})
28-
Base.precompile(Tuple{Core.kwftype(typeof(innerjoin)),NamedTuple{(:on,), Tuple{Symbol}},typeof(innerjoin),DataFrame,DataFrame})
29-
Base.precompile(Tuple{Core.kwftype(typeof(leftjoin!)),NamedTuple{(:on, :makeunique, :source), Tuple{Pair{Symbol, Symbol}, Bool, Symbol}},typeof(leftjoin!),DataFrame,DataFrame})
30-
Base.precompile(Tuple{Core.kwftype(typeof(leftjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Vector{Any}, Bool, Pair{Bool, Bool}, Pair{Symbol, String}}},typeof(leftjoin),DataFrame,DataFrame})
31-
Base.precompile(Tuple{Core.kwftype(typeof(leftjoin)),NamedTuple{(:on, :renamecols, :source), Tuple{Vector{Any}, Pair{String, String}, Symbol}},typeof(leftjoin),DataFrame,DataFrame})
32-
Base.precompile(Tuple{Core.kwftype(typeof(outerjoin)),NamedTuple{(:on, :makeunique, :validate, :renamecols), Tuple{Vector{Any}, Bool, Pair{Bool, Bool}, Pair{Symbol, Symbol}}},typeof(outerjoin),DataFrame,DataFrame})
33-
Base.precompile(Tuple{Core.kwftype(typeof(outerjoin)),NamedTuple{(:on, :source, :makeunique), Tuple{Symbol, String, Bool}},typeof(outerjoin),DataFrame,DataFrame})
34-
Base.precompile(Tuple{Core.kwftype(typeof(outerjoin)),NamedTuple{(:on, :source, :makeunique), Tuple{Symbol, Symbol, Bool}},typeof(outerjoin),DataFrame,DataFrame})
35-
Base.precompile(Tuple{Core.kwftype(typeof(reduce)),NamedTuple{(:cols, :source), Tuple{Symbol, Nothing}},typeof(reduce),typeof(vcat),NTuple{5, DataFrame}})
36-
Base.precompile(Tuple{Core.kwftype(typeof(select!)),NamedTuple{(:renamecols,), Tuple{Bool}},typeof(select!),GroupedDataFrame{SubDataFrame{DataFrame, SubIndex{Index, Vector{Int}, Vector{Int}}, UnitRange{Int}}},Union{Regex, AbstractString, Function, Signed, Symbol, Unsigned, Pair, Type, All, Between, Cols, InvertedIndex, AbstractVecOrMat{T} where T}})
37-
Base.precompile(Tuple{Core.kwftype(typeof(show)),NamedTuple{(:allrows, :allcols, :allgroups, :rowlabel, :summary, :truncate), Tuple{Bool, Bool, Bool, Symbol, Bool, Int}},typeof(show),Base.PipeEndpoint,GroupedDataFrame{DataFrame}})
38-
Base.precompile(Tuple{Core.kwftype(typeof(show)),NamedTuple{(:show_row_number,), Tuple{Bool}},typeof(show),IOContext{IOBuffer},DataFrame})
39-
Base.precompile(Tuple{Core.kwftype(typeof(vcat)),NamedTuple{(:cols, :source), Tuple{Symbol, Symbol}},typeof(vcat),DataFrame,Vararg{DataFrame}})
40-
Base.precompile(Tuple{Core.kwftype(typeof(vcat)),NamedTuple{(:cols,), Tuple{Vector{Symbol}}},typeof(vcat),DataFrame,Vararg{DataFrame}})
41-
Base.precompile(Tuple{Type{DataFrame},Vector{Tuple{Int, Int}}})
42-
Base.precompile(Tuple{typeof(_innerjoin_sorted),OnCol{Tuple{Vector{Int}, Vector{String}}, 2},OnCol{Tuple{SubArray{Union{Missing, Int}, 1, Vector{Union{Missing, Int}}, Tuple{Vector{Int}}, false}, SubArray{Union{Missing, String}, 1, Vector{Union{Missing, String}}, Tuple{Vector{Int}}, false}}, 2}})
43-
Base.precompile(Tuple{typeof(_innerjoin_sorted),OnCol{Tuple{Vector{UInt32}, Vector{UInt32}, Vector{UInt32}}, 3},OnCol{Tuple{Vector{UInt32}, Vector{UInt32}, Vector{UInt32}}, 3}})
44-
Base.precompile(Tuple{typeof(_innerjoin_unsorted_int),SubArray{Union{Missing, Int}, 1, Vector{Union{Missing, Int}}, Tuple{Vector{Int}}, false},Vector{Missing}})
45-
Base.precompile(Tuple{typeof(_innerjoin_unsorted_int),Vector{Int32},Vector{Int32}})
46-
Base.precompile(Tuple{typeof(_innerjoin_unsorted),OnCol{Tuple{Vector{Union{Missing, String}}, Vector{Union{Missing, String}}, Vector{Union{Missing, String}}}, 3},OnCol{Tuple{Vector{String}, Vector{String}, Vector{String}}, 3}})
47-
Base.precompile(Tuple{typeof(_innerjoin_unsorted),OnCol{Tuple{Vector{Union{Missing, Symbol}}, Vector{Union{Missing, Symbol}}}, 2},OnCol{Tuple{Vector{Union{Missing, Symbol}}, Vector{Union{Missing, Symbol}}}, 2}})
48-
Base.precompile(Tuple{typeof(_mean_fast),Vector{Vector{Union{Missing, Int}}}})
49-
Base.precompile(Tuple{typeof(_semijoin_unsorted_int),Vector{Int},Vector{Union{Missing, Int}},BitVector,Bool})
50-
Base.precompile(Tuple{typeof(_semijoin_unsorted),OnCol{Tuple{Vector{UInt32}, Vector{UInt32}}, 2},OnCol{Tuple{Vector{Union{Missing, UInt32}}, Vector{Union{Missing, UInt32}}}, 2},BitVector,Bool})
51-
Base.precompile(Tuple{typeof(Base.Broadcast.dotview),SubDataFrame{DataFrame, Index, Vector{Int}},Colon,Symbol})
52-
Base.precompile(Tuple{typeof(describe),DataFrame})
53-
Base.precompile(Tuple{typeof(do_call),ComposedFunction{typeof(sum), typeof(skipmissing)},Vector{Int},Vector{Int},Vector{Int},GroupedDataFrame{DataFrame},Tuple{Vector{Union{Missing, String}}},Int})
54-
Base.precompile(Tuple{typeof(do_call),typeof(cor),Vector{Int},Vector{Int},Vector{Int},GroupedDataFrame{DataFrame},Tuple{Vector{Int}, Vector{Int}},Int})
55-
Base.precompile(Tuple{typeof(do_call),typeof(sum),Vector{Int},Vector{Int},Vector{Int},GroupedDataFrame{DataFrame},Tuple{Vector{DataFrame}},Int})
56-
Base.precompile(Tuple{typeof(groupby),DataFrame,Symbol})
57-
Base.precompile(Tuple{typeof(names),DataFrameRow{DataFrame, SubIndex{Index, Vector{Int}, Vector{Int}}},Cols{Tuple{String}}})
58-
Base.precompile(Tuple{typeof(permutedims),DataFrame,Symbol})
59-
Base.precompile(Tuple{typeof(prepare_on_col),Vector{Int},Vector{Union{}}})
60-
Base.precompile(Tuple{typeof(repeat),DataFrame,Int})
61-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{BitVector},Val{false},Vector{Int},Bool,Nothing})
62-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{PooledVector{Int, UInt32, Vector{UInt32}}, PooledVector{String, UInt32, Vector{UInt32}}},Val{false},Vector{Int},Bool,Nothing})
63-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{PooledVector{String, UInt32, Vector{UInt32}}, Vector{Int}},Val{false},Vector{Int},Bool,Nothing})
64-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{PooledVector{String, UInt32, Vector{UInt32}}, Vector{Int32}},Val{false},Vector{Int},Bool,Nothing})
65-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{PooledVector{Union{Missing, Int}, UInt32, Vector{UInt32}}, PooledVector{String, UInt32, Vector{UInt32}}},Val{false},Vector{Int},Bool,Nothing})
66-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{PooledVector{Union{Missing, String}, UInt32, Vector{UInt32}}, PooledVector{String, UInt32, Vector{UInt32}}},Val{false},Vector{Int},Bool,Nothing})
67-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{RepeatedVector{Union{Missing, Int}}, RepeatedVector{Union{Missing, Int}}, RepeatedVector{Union{Missing, String}}, Vector{Union{Missing, Int}}},Val{false},Vector{Int},Bool,Nothing})
68-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{SubArray{Int, 1, Vector{Int}, Tuple{Base.OneTo{Int}}, true}},Tuple{IntegerRefpool{Int}},Tuple{IntegerRefarray{SubArray{Int, 1, Vector{Int}, Tuple{Base.OneTo{Int}}, true}}},Val{false},Vector{Int},Bool,Bool})
69-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{SubArray{Union{Missing, Int}, 1, Vector{Union{Missing, Int}}, Tuple{Base.OneTo{Int}}, true}},Tuple{IntegerRefpool{Union{Missing, Int}}},Tuple{IntegerRefarray{SubArray{Union{Missing, Int}, 1, Vector{Union{Missing, Int}}, Tuple{Base.OneTo{Int}}, true}}},Val{false},Vector{Int},Bool,Bool})
70-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Bool}},Val{false},Vector{Int},Bool,Nothing})
71-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Float64}, Vector{Int}},Val{false},Vector{Int},Bool,Nothing})
72-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Int}, Vector{Int}, Vector{Int}},Val{false},Vector{Int},Bool,Nothing})
73-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Int}, Vector{Union{Missing, Int}}},Val{false},Vector{Int},Bool,Nothing})
74-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Int32}, Vector{Int32}},Val{false},Vector{Int},Bool,Nothing})
75-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Int32}},Tuple{IntegerRefpool{Int}},Tuple{IntegerRefarray{Vector{Int32}}},Val{false},Vector{Int},Bool,Bool})
76-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Int32}},Val{false},Vector{Int},Bool,Nothing})
77-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{String}, Vector{Int32}},Val{false},Vector{Int},Bool,Nothing})
78-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, BigInt}}},Val{false},Vector{Int},Bool,Nothing})
79-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Bool}}},Tuple{IntegerRefpool{Union{Missing, Int}}},Tuple{IntegerRefarray{Vector{Union{Missing, Bool}}}},Val{false},Vector{Int},Bool,Bool})
80-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Bool}}},Val{false},Vector{Int},Bool,Nothing})
81-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Float64}}, Vector{Float64}},Val{false},Vector{Int},Bool,Nothing})
82-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Float64}}, Vector{Int}},Val{false},Vector{Int},Bool,Nothing})
83-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Int}}, Vector{Union{Missing, Float64}}},Tuple{IntegerRefpool{Union{Missing, Int}}, IntegerRefpool{Union{Missing, Int}}},Tuple{IntegerRefarray{Vector{Union{Missing, Int}}}, IntegerRefarray{Vector{Union{Missing, Float64}}}},Val{false},Vector{Int},Bool,Bool})
84-
Base.precompile(Tuple{typeof(row_group_slots),Tuple{Vector{Union{Missing, Int}}},Val{false},Vector{Int},Bool,Nothing})
85-
Base.precompile(Tuple{typeof(select),SubDataFrame{DataFrame, Index, UnitRange{Int}},Any})
86-
Base.precompile(Tuple{typeof(select),SubDataFrame{DataFrame, SubIndex{Index, UnitRange{Int}, UnitRange{Int}}, Vector{Int}},Any,Any,Vararg{Any}})
87-
Base.precompile(Tuple{typeof(setindex!),DataFrame,Vector{Int},Vector{Bool},Int})
88-
Base.precompile(Tuple{typeof(show),IOBuffer,DataFrameRow{DataFrame, SubIndex{Index, UnitRange{Int}, UnitRange{Int}}}})
89-
Base.precompile(Tuple{typeof(sort),SubDataFrame{DataFrame, SubIndex{Index, UnitRange{Int}, UnitRange{Int}}, UnitRange{Int}}})
90-
Base.precompile(Tuple{typeof(subset),GroupedDataFrame{SubDataFrame{DataFrame, Index, UnitRange{Int}}},Any})
91-
Base.precompile(Tuple{typeof(transform),GroupedDataFrame{DataFrame},Union{Regex, AbstractString, Function, Signed, Symbol, Unsigned, Pair, Type, All, Between, Cols, InvertedIndex, AbstractVecOrMat{T} where T}})
92-
Base.precompile(Tuple{typeof(view),SubDataFrame{DataFrame, Index, UnitRange{Int}},Int,Int})
93-
94-
if all
95-
for v in ([1, 2], [2, 1], [2, 2, 1], Int32[1, 2], Int32[2, 1], Int32[2, 2, 1]),
96-
op in (identity, x -> string.(x), x -> PooledArrays.PooledArray(string.(x))),
97-
on in (:v1, [:v1, :v2])
98-
df = DataFrame(v1=op(v), v2=v)
99-
combine(groupby(df, on), identity, :v1 => identity,
100-
:v2 => ByRow(identity), :v2 => sum)
101-
innerjoin(df, select(df, on), on=on)
102-
outerjoin(df, select(df, on), on=on)
103-
end
104-
end
105-
return nothing
3+
SnoopPrecompile.@precompile_all_calls begin
4+
df = DataFrame(a=[2, 5, 3, 1, 0], b=["a", "b", "c", "a", "b"], c=1:5,
5+
p=PooledArray(["a", "b", "c", "a", "b"]),
6+
q=[true, false, true, false, true],
7+
f=Float64[2, 5, 3, 1, 0])
8+
describe(df)
9+
names(df[1, 1:2])
10+
sort(df, :a)
11+
combine(df, :c, [:c :f] .=> [sum, mean, std], :c => :d, [:a, :c] => cor)
12+
transform(df, :c, [:c :f] .=> [sum, mean, std], :c => :d, [:a, :c] => cor)
13+
groupby(df, :a)
14+
groupby(df, :q)
15+
groupby(df, :p)
16+
gdf = groupby(df, :b)
17+
combine(gdf, :c, [:c :f] .=> [sum, mean, std], :c => :d, [:a, :c] => cor)
18+
transform(gdf, :c, [:c :f] .=> [sum, mean, std], :c => :d, [:a, :c] => cor)
19+
innerjoin(df, df, on=:a, makeunique=true)
20+
innerjoin(df, df, on=:b, makeunique=true)
21+
innerjoin(df, df, on=:c, makeunique=true)
22+
outerjoin(df, df, on=:a, makeunique=true)
23+
outerjoin(df, df, on=:b, makeunique=true)
24+
outerjoin(df, df, on=:c, makeunique=true)
25+
semijoin(df, df, on=:a)
26+
semijoin(df, df, on=:b)
27+
semijoin(df, df, on=:c)
28+
leftjoin!(df, DataFrame(a=[2, 5, 3, 1, 0]), on=:a)
29+
leftjoin!(df, DataFrame(b=["a", "b", "c", "d", "e"]), on=:b)
30+
leftjoin!(df, DataFrame(c=1:5), on=:c)
31+
reduce(vcat, [df, df])
32+
show(IOBuffer(), df)
33+
subset(df, :q)
34+
@view df[1:3, :]
35+
@view df[:, 1:2]
36+
transform!(df, :c, [:c :f] .=> [sum, mean, std], :c => :d, [:a, :c] => cor)
10637
end

‎test/precompile.jl‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎test/runtests.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ my_tests = ["utils.jl",
5656
"string.jl",
5757
"multithreading.jl",
5858
"metadata.jl",
59-
"precompile.jl",
6059
"deprecated.jl"]
6160

6261
println("Running tests:")

0 commit comments

Comments
 (0)