From 84f14f1dc4699841e13f72224e92634ef06becbe Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 17:12:23 +0000 Subject: [PATCH 01/16] feat: abstract types for Pareto front elements --- src/Core.jl | 3 +++ src/HallOfFame.jl | 39 ++++++++++++++++++++++++++++++++++++++- src/OptionsStruct.jl | 16 ++++++++++++++++ src/SymbolicRegression.jl | 3 +++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/Core.jl b/src/Core.jl index bc9210aed..62d2e7732 100644 --- a/src/Core.jl +++ b/src/Core.jl @@ -17,6 +17,9 @@ using .OptionsStructModule: AbstractOptions, Options, ComplexityMapping, + AbstractParetoOptions, + ParetoSingleOptions, + ParetoNeighborhoodOptions, specialized_options, operator_specialization using .OperatorsModule: diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 7d870f78b..e31163c87 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -3,6 +3,7 @@ module HallOfFameModule using StyledStrings: @styled_str using DynamicExpressions: AbstractExpression, string_tree using ..UtilsModule: split_string, AnnotatedIOBuffer, dump_buffer +using ..CoreModule: ParetoSingleOptions, ParetoNeighborhoodOptions using ..CoreModule: AbstractOptions, Dataset, DATA_TYPE, LOSS_TYPE, relu, create_expression using ..ComplexityModule: compute_complexity using ..PopMemberModule: PopMember @@ -10,7 +11,43 @@ using ..InterfaceDynamicExpressionsModule: format_dimensions, WILDCARD_UNIT_STRI using Printf: @sprintf """ - HallOfFame{T<:DATA_TYPE,L<:LOSS_TYPE} + AbstractParetoElement{P<:PopMember} + +Abstract type for storing elements on the Pareto frontier. + +# Subtypes +- `ParetoSingle`: Stores a single member at each complexity level +- `ParetoNeighborhood`: Stores multiple members at each complexity level in a fixed-size bucket +""" +abstract type AbstractParetoElement{P<:PopMember} end + +struct ParetoSingle{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} + member::P +end +struct ParetoNeighborhood{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} + members::Vector{P} + max_size::Int +end + +function _depwarn_pareto_single(funcsym::Symbol) + return Base.depwarn( + "Hall of fame `.members` is now a vector of `AbstractParetoElement` objects. ", + funcsym, + ) +end + +function Base.getproperty(s::ParetoSingle, name::Symbol) + name == :member && return getfield(s, :member) + _depwarn_pareto_single(:getproperty) + return getproperty(s.member, name) +end +function Base.setproperty!(s::ParetoSingle, name::Symbol, value) + name == :member && return setfield!(s, :member, value) + _depwarn_pareto_single(:setproperty!) + return setproperty!(s.member, name, value) +end + +""" List of the best members seen all time in `.members`, with `.members[c]` being the best member seen at complexity c. Including only the members which actually diff --git a/src/OptionsStruct.jl b/src/OptionsStruct.jl index 169b30574..a616da82f 100644 --- a/src/OptionsStruct.jl +++ b/src/OptionsStruct.jl @@ -8,6 +8,22 @@ using LossFunctions: SupervisedLoss import ..MutationWeightsModule: AbstractMutationWeights +""" + AbstractParetoOptions + +Abstract type for different Pareto front storage strategies. + +# Subtypes +- `ParetoSingleOptions`: Store only the single best member at each complexity +- `ParetoNeighborhoodOptions`: Store multiple members at each complexity in a fixed-size bucket +""" +abstract type AbstractParetoOptions end + +struct ParetoSingleOptions <: AbstractParetoOptions end +struct ParetoNeighborhoodOptions <: AbstractParetoOptions + bucket_size::Int +end + """ This struct defines how complexity is calculated. diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index fd95b0851..460f2b6df 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -234,6 +234,9 @@ using .CoreModule: ComplexityMapping, AbstractMutationWeights, MutationWeights, + AbstractParetoOptions, + ParetoSingleOptions, + ParetoNeighborhoodOptions, get_safe_op, max_features, is_weighted, From c3c0550c377de10eff5c8889bc2dec24264ad97c Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 19:52:51 +0000 Subject: [PATCH 02/16] feat: working abstract pareto element --- src/ExpressionBuilder.jl | 37 +++++++-- src/HallOfFame.jl | 157 +++++++++++++++++++++++++++++--------- src/MLJInterface.jl | 9 ++- src/Options.jl | 7 +- src/OptionsStruct.jl | 2 + src/SearchUtils.jl | 18 ----- src/SingleIteration.jl | 19 +++-- src/SymbolicRegression.jl | 20 +++-- 8 files changed, 190 insertions(+), 79 deletions(-) diff --git a/src/ExpressionBuilder.jl b/src/ExpressionBuilder.jl index 00264be6a..34479835e 100644 --- a/src/ExpressionBuilder.jl +++ b/src/ExpressionBuilder.jl @@ -10,7 +10,7 @@ using DynamicExpressions: AbstractExpressionNode, AbstractExpression, constructorof, with_metadata using StatsBase: StatsBase using ..CoreModule: AbstractOptions, Dataset -using ..HallOfFameModule: HallOfFame +using ..HallOfFameModule: HallOfFame, ParetoSingle, ParetoNeighborhood using ..PopulationModule: Population using ..PopMemberModule: PopMember @@ -124,20 +124,33 @@ end pop::Population, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L} return Population( - map(Fix{2}(Fix{3}(embed_metadata, dataset), options), pop.members) + map(member -> embed_metadata(member, options, dataset), pop.members) + ) + end + function embed_metadata( + el::ParetoSingle, options::AbstractOptions, dataset::Dataset{T,L} + ) where {T,L} + return ParetoSingle(embed_metadata(el.member, options, dataset)) + end + function embed_metadata( + el::ParetoNeighborhood, options::AbstractOptions, dataset::Dataset{T,L} + ) where {T,L} + return ParetoNeighborhood( + map(member -> embed_metadata(member, options, dataset), el.members), + el.bucket_size, ) end function embed_metadata( hof::HallOfFame, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L} return HallOfFame( - map(Fix{2}(Fix{3}(embed_metadata, dataset), options), hof.members), hof.exists + map(el -> embed_metadata(el, options, dataset), hof.elements), hof.exists ) end function embed_metadata( - vec::Vector{H}, options::AbstractOptions, dataset::Dataset{T,L} + sets::Vector{H}, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L,H<:Union{HallOfFame,Population,PopMember}} - return map(Fix{2}(Fix{3}(embed_metadata, dataset), options), vec) + return map(set -> embed_metadata(set, options, dataset), sets) end end @@ -171,11 +184,23 @@ function strip_metadata( ) where {T,L} return Population(map(member -> strip_metadata(member, options, dataset), pop.members)) end +function strip_metadata( + el::ParetoSingle, options::AbstractOptions, dataset::Dataset{T,L} +) where {T,L} + return ParetoSingle(strip_metadata(el.member, options, dataset)) +end +function strip_metadata( + el::ParetoNeighborhood, options::AbstractOptions, dataset::Dataset{T,L} +) where {T,L} + return ParetoNeighborhood( + map(member -> strip_metadata(member, options, dataset), el.members), el.bucket_size + ) +end function strip_metadata( hof::HallOfFame, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L} return HallOfFame( - map(member -> strip_metadata(member, options, dataset), hof.members), hof.exists + map(el -> strip_metadata(el, options, dataset), hof.elements), hof.exists ) end diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index e31163c87..d6840544f 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -2,13 +2,15 @@ module HallOfFameModule using StyledStrings: @styled_str using DynamicExpressions: AbstractExpression, string_tree +# using DataStructures: PriorityQueue +using Printf: @sprintf using ..UtilsModule: split_string, AnnotatedIOBuffer, dump_buffer using ..CoreModule: ParetoSingleOptions, ParetoNeighborhoodOptions using ..CoreModule: AbstractOptions, Dataset, DATA_TYPE, LOSS_TYPE, relu, create_expression using ..ComplexityModule: compute_complexity using ..PopMemberModule: PopMember using ..InterfaceDynamicExpressionsModule: format_dimensions, WILDCARD_UNIT_STRING -using Printf: @sprintf +using ..PopulationModule: Population """ AbstractParetoElement{P<:PopMember} @@ -21,14 +23,27 @@ Abstract type for storing elements on the Pareto frontier. """ abstract type AbstractParetoElement{P<:PopMember} end +pop_member_type(::Type{<:AbstractParetoElement{P}}) where {P} = P + struct ParetoSingle{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} member::P end struct ParetoNeighborhood{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} members::Vector{P} - max_size::Int + bucket_size::Int end +Base.copy(el::ParetoSingle) = ParetoSingle(copy(el.member)) +Base.copy(el::ParetoNeighborhood) = ParetoNeighborhood(copy(el.members), el.bucket_size) + +Base.first(el::ParetoSingle) = el.member +Base.first(el::ParetoNeighborhood) = first(el.members) + +Base.iterate(el::ParetoSingle) = (el.member, nothing) +Base.iterate(::ParetoSingle, ::Nothing) = nothing +Base.iterate(el::ParetoNeighborhood) = iterate(el.members) +Base.iterate(el::ParetoNeighborhood, state) = iterate(el.members, state) + function _depwarn_pareto_single(funcsym::Symbol) return Base.depwarn( "Hall of fame `.members` is now a vector of `AbstractParetoElement` objects. ", @@ -48,6 +63,7 @@ function Base.setproperty!(s::ParetoSingle, name::Symbol, value) end """ + HallOfFame{T<:DATA_TYPE,L<:LOSS_TYPE,N<:AbstractExpression{T}} List of the best members seen all time in `.members`, with `.members[c]` being the best member seen at complexity c. Including only the members which actually @@ -59,9 +75,25 @@ have been set, you can run `.members[exists]`. These are ordered by complexity, with `.members[1]` the member with complexity 1. - `exists::Array{Bool,1}`: Whether the member at the given complexity has been set. """ -struct HallOfFame{T<:DATA_TYPE,L<:LOSS_TYPE,N<:AbstractExpression{T}} - members::Array{PopMember{T,L,N},1} - exists::Array{Bool,1} #Whether it has been set +struct HallOfFame{ + T<:DATA_TYPE, + L<:LOSS_TYPE, + N<:AbstractExpression{T}, + H<:AbstractParetoElement{<:PopMember{T,L,N}}, +} + elements::Vector{H} + exists::Vector{Bool} +end +pop_member_type(::Type{<:HallOfFame{T,L,N,H}}) where {T,L,N,H} = pop_member_type(H) +function Base.getproperty(hof::HallOfFame, name::Symbol) + if name == :members + Base.depwarn( + "HallOfFame.members is deprecated. Use HallOfFame.elements instead.", + :getproperty, + ) + return getfield(hof, :elements) + end + return getfield(hof, name) end function Base.show(io::IO, mime::MIME"text/plain", hof::HallOfFame{T,L,N}) where {T,L,N} println(io, "HallOfFame{...}:") @@ -98,58 +130,109 @@ Arguments: - `dataset`: Dataset containing the input data. """ function HallOfFame( - options::AbstractOptions, dataset::Dataset{T,L} + options::AbstractOptions, dataset::Dataset{T,L}; ) where {T<:DATA_TYPE,L<:LOSS_TYPE} base_tree = create_expression(zero(T), options, dataset) + N = typeof(base_tree) + member = PopMember( + base_tree, L(0), L(Inf), options; parent=-1, deterministic=options.deterministic + ) - return HallOfFame{T,L,typeof(base_tree)}( - [ - PopMember( - copy(base_tree), - L(0), - L(Inf), - options; - parent=-1, - deterministic=options.deterministic, - ) for i in 1:(options.maxsize) - ], + return HallOfFame( + [init_element(options.pareto_element_options, member) for i in 1:(options.maxsize)], [false for i in 1:(options.maxsize)], ) end +Base.copy(hof::HallOfFame) = HallOfFame(map(copy, hof.members), copy(hof.exists)) -function Base.copy(hof::HallOfFame) - return HallOfFame( - [copy(member) for member in hof.members], [exists for exists in hof.exists] - ) +function init_element(::Union{ParetoSingleOptions,ParetoSingle}, member::PopMember) + return ParetoSingle(copy(member)) +end +function init_element( + opt::Union{ParetoNeighborhoodOptions,ParetoNeighborhood}, member::PopMember +) + return ParetoNeighborhood([copy(member)], opt.bucket_size) +end + +function Base.push!(hof::HallOfFame, (size, member)::Pair{<:Integer,<:PopMember}) + maxsize = length(hof.elements) + if 0 < size <= maxsize + if !hof.exists[size] + hof.elements[size] = init_element(hof.elements[size], member) + hof.exists[size] = true + else + hof.elements[size] = push!(hof.elements[size], member.score => member) + end + end + return hof +end + +function Base.push!(el::ParetoSingle, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) + return el.member.score > score ? ParetoSingle(copy(member)) : el +end +function Base.push!(el::ParetoNeighborhood, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) + return error("Not implemented") +end + +function Base.push!(hof::HallOfFame, pop::Population; options::AbstractOptions) + for member in pop.members + size = compute_complexity(member, options) + push!(hof, size => member) + end + return hof +end + +function Base.merge!(hof1::HallOfFame, hof2::HallOfFame) + for i in eachindex(hof1.elements, hof1.exists, hof2.elements, hof2.exists) + if hof1.exists[i] && hof2.exists[i] + hof1.elements[i] = merge(hof1.elements[i], hof2.elements[i]) + elseif !hof1.exists[i] && hof2.exists[i] + hof1.elements[i] = copy(hof2.elements[i]) + hof1.exists[i] = true + else + # do nothing, as !hof2.exists[i] + end + end + return hof1 +end +function Base.merge(el1::ParetoSingle, el2::ParetoSingle) + # Remember: we want the MIN score (bad API choice, but we're stuck with it for now) + return el1.member.score <= el2.member.score ? el1 : el2 +end +function Base.merge(el1::ParetoNeighborhood, el2::ParetoNeighborhood) + return error("Not implemented") end """ - calculate_pareto_frontier(hallOfFame::HallOfFame{T,L,P}) where {T<:DATA_TYPE,L<:LOSS_TYPE} + calculate_pareto_frontier(hof::HallOfFame) """ -function calculate_pareto_frontier(hallOfFame::HallOfFame{T,L,N}) where {T,L,N} - # TODO - remove dataset from args. - P = PopMember{T,L,N} +function calculate_pareto_frontier(hof::HallOfFame) + P = pop_member_type(typeof(hof)) # Dominating pareto curve - must be better than all simpler equations dominating = P[] - for size in eachindex(hallOfFame.members) - if !hallOfFame.exists[size] + for i in eachindex(hof.elements) + if !hof.exists[i] continue end - member = hallOfFame.members[size] - # We check if this member is better than all members which are smaller than it and - # also exist. - betterThanAllSmaller = true - for i in 1:(size - 1) - if !hallOfFame.exists[i] + element = hof.elements[i] + member = first(element) + # We check if this member is better than all + # elements which are smaller than it and also exist. + is_dominating = true + for j in 1:(i - 1) + if !hof.exists[j] continue end - simpler_member = hallOfFame.members[i] - if member.loss >= simpler_member.loss - betterThanAllSmaller = false + smaller_element = hof.elements[j] + smaller_member = first(smaller_element) + if member.loss >= smaller_member.loss + is_dominating = false break end + # TODO: Why are we using loss and not score? In other words, + # why are we _pushing_ based on score and not loss? end - if betterThanAllSmaller + if is_dominating push!(dominating, copy(member)) end end diff --git a/src/MLJInterface.jl b/src/MLJInterface.jl index 2c37c4583..33def62bc 100644 --- a/src/MLJInterface.jl +++ b/src/MLJInterface.jl @@ -27,7 +27,14 @@ using LossFunctions: SupervisedLoss using ..InterfaceDynamicQuantitiesModule: get_dimensions_type using ..InterfaceDynamicExpressionsModule: InterfaceDynamicExpressionsModule as IDE using ..CoreModule: - Options, Dataset, AbstractMutationWeights, MutationWeights, LOSS_TYPE, ComplexityMapping + Options, + Dataset, + AbstractMutationWeights, + MutationWeights, + LOSS_TYPE, + ComplexityMapping, + AbstractParetoOptions, + ParetoSingleOptions using ..CoreModule.OptionsModule: DEFAULT_OPTIONS, OPTION_DESCRIPTIONS using ..ComplexityModule: compute_complexity using ..HallOfFameModule: HallOfFame, format_hall_of_fame diff --git a/src/Options.jl b/src/Options.jl index b6f1dd847..afaa256a6 100644 --- a/src/Options.jl +++ b/src/Options.jl @@ -29,7 +29,8 @@ using ..OperatorsModule: safe_atanh using ..MutationWeightsModule: AbstractMutationWeights, MutationWeights, mutations import ..OptionsStructModule: Options -using ..OptionsStructModule: ComplexityMapping, operator_specialization +using ..OptionsStructModule: + ComplexityMapping, operator_specialization, AbstractParetoOptions, ParetoSingleOptions using ..UtilsModule: @save_kwargs, @ignore """Build constraints on operator-level complexity from a user-passed dict.""" @@ -523,6 +524,7 @@ $(OPTION_DESCRIPTIONS) ### hof_migration ### fraction_replaced ### fraction_replaced_hof + ### pareto_element_options ### topn ## 9. Data Preprocessing: ### [none] @@ -581,6 +583,7 @@ $(OPTION_DESCRIPTIONS) hof_migration::Bool=true, fraction_replaced::Union{Real,Nothing}=nothing, fraction_replaced_hof::Union{Real,Nothing}=nothing, + pareto_element_options::AbstractParetoOptions=ParetoSingleOptions(), topn::Union{Nothing,Integer}=nothing, ## 9. Data Preprocessing: ## 10. Stopping Criteria: @@ -873,6 +876,7 @@ $(OPTION_DESCRIPTIONS) expression_type, typeof(expression_options), typeof(set_mutation_weights), + typeof(pareto_element_options), turbo, bumper, deprecated_return_state::Union{Bool,Nothing}, @@ -913,6 +917,7 @@ $(OPTION_DESCRIPTIONS) ncycles_per_iteration, fraction_replaced, fraction_replaced_hof, + pareto_element_options, topn, verbosity, Val(print_precision), diff --git a/src/OptionsStruct.jl b/src/OptionsStruct.jl index a616da82f..d2bd777de 100644 --- a/src/OptionsStruct.jl +++ b/src/OptionsStruct.jl @@ -200,6 +200,7 @@ struct Options{ E<:AbstractExpression, EO<:NamedTuple, MW<:AbstractMutationWeights, + PO<:AbstractParetoOptions, _turbo, _bumper, _return_state, @@ -240,6 +241,7 @@ struct Options{ ncycles_per_iteration::Int fraction_replaced::Float32 fraction_replaced_hof::Float32 + pareto_element_options::PO topn::Int verbosity::Union{Int,Nothing} v_print_precision::Val{print_precision} diff --git a/src/SearchUtils.jl b/src/SearchUtils.jl index 89ca03ee2..fa9effc41 100644 --- a/src/SearchUtils.jl +++ b/src/SearchUtils.jl @@ -676,22 +676,4 @@ function construct_datasets( ] end -function update_hall_of_fame!( - hall_of_fame::HallOfFame, members::Vector{PM}, options::AbstractOptions -) where {PM<:PopMember} - for member in members - size = compute_complexity(member, options) - valid_size = 0 < size <= options.maxsize - if !valid_size - continue - end - not_filled = !hall_of_fame.exists[size] - better_than_current = member.score < hall_of_fame.members[size].score - if not_filled || better_than_current - hall_of_fame.members[size] = copy(member) - hall_of_fame.exists[size] = true - end - end -end - end diff --git a/src/SingleIteration.jl b/src/SingleIteration.jl index 2d36e6c87..a7c2bd9fc 100644 --- a/src/SingleIteration.jl +++ b/src/SingleIteration.jl @@ -56,9 +56,9 @@ function s_r_cycle( num_evals += tmp_num_evals for (i, member) in enumerate(pop.members) size = compute_complexity(member, options) - score = if options.batching + if options.batching oid = member.tree - if loss_cache[i].oid != oid || first_loop + same_batch_score = if loss_cache[i].oid != oid || first_loop # Evaluate on fixed batch so that we can more accurately # compare expressions with a batched loss (though the batch # changes each iteration, and we evaluate on full-batch outside, @@ -73,8 +73,14 @@ function s_r_cycle( # the cached score loss_cache[i].score end + + stored_member = copy(member) + stored_member.score = same_batch_score + # ^I think we don't want to _store_ this score in this member + # since it would fundamentally mutate the population + push!(best_examples_seen, size => stored_member) else - member.score + push!(best_examples_seen, size => member) end # TODO: Note that this per-population hall of fame only uses the batched # loss, and is therefore inaccurate. Therefore, some expressions @@ -83,13 +89,6 @@ function s_r_cycle( # - Could just recompute losses here (expensive) # - Average over a few batches # - Store multiple expressions in hall of fame - if 0 < size <= options.maxsize && ( - !best_examples_seen.exists[size] || - score < best_examples_seen.members[size].score - ) - best_examples_seen.exists[size] = true - best_examples_seen.members[size] = copy(member) - end end first_loop = false end diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index 460f2b6df..e6d305c4c 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -320,7 +320,6 @@ using .SearchUtilsModule: construct_datasets, save_to_file, get_cur_maxsize, - update_hall_of_fame!, logging_callback! using .LoggingModule: AbstractSRLogger, SRLogger, get_logger using .TemplateExpressionModule: TemplateExpression, TemplateStructure @@ -691,12 +690,20 @@ function _initialize_search!( # case the dataset changed: for j in eachindex(init_hall_of_fame, datasets, state.halls_of_fame) hof = strip_metadata(init_hall_of_fame[j], options, datasets[j]) - for member in hof.members[hof.exists] - score, result_loss = score_func(datasets[j], member, options) + new_hof = HallOfFame(options, datasets[j]) + for el in hof.elements[hof.exists], member in el + size = compute_complexity(member, options) + score, result_loss = score_func( + datasets[j], member, options; complexity=size + ) member.score = score member.loss = result_loss + + # In case the new loss changes the elements stored in the Pareto + # frontier, we push them individually: + push!(new_hof, size => member) end - state.halls_of_fame[j] = hof + state.halls_of_fame[j] = new_hof end end @@ -886,9 +893,10 @@ function _main_search_loop!( update_frequencies!(state.all_running_search_statistics[j]; size) end #! format: off - update_hall_of_fame!(state.halls_of_fame[j], cur_pop.members, options) - update_hall_of_fame!(state.halls_of_fame[j], best_seen.members[best_seen.exists], options) + push!(state.halls_of_fame[j], cur_pop; options) + merge!(state.halls_of_fame[j], best_seen) #! format: on + # TODO: Confirm that `best_seen.members` have full-batch scores # Dominating pareto curve - must be better than all simpler equations dominating = calculate_pareto_frontier(state.halls_of_fame[j]) From 91b485df66f61d0b470366e94065e10ee1d87991 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 19:57:35 +0000 Subject: [PATCH 03/16] fix: missing copy in `merge` --- src/HallOfFame.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index d6840544f..953696907 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -197,7 +197,7 @@ function Base.merge!(hof1::HallOfFame, hof2::HallOfFame) end function Base.merge(el1::ParetoSingle, el2::ParetoSingle) # Remember: we want the MIN score (bad API choice, but we're stuck with it for now) - return el1.member.score <= el2.member.score ? el1 : el2 + return el1.member.score <= el2.member.score ? el1 : copy(el2) end function Base.merge(el1::ParetoNeighborhood, el2::ParetoNeighborhood) return error("Not implemented") From 5911382651c4e242c08dafb5ea3278e046f8ae1f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 19:58:15 +0000 Subject: [PATCH 04/16] docs: improve docstring --- src/HallOfFame.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 953696907..3620db6f1 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -205,10 +205,11 @@ end """ calculate_pareto_frontier(hof::HallOfFame) + +Compute the dominating pareto curve - each returned member must be better than all simpler equations. """ function calculate_pareto_frontier(hof::HallOfFame) P = pop_member_type(typeof(hof)) - # Dominating pareto curve - must be better than all simpler equations dominating = P[] for i in eachindex(hof.elements) if !hof.exists[i] From b740e3a4bcf24a8dc00c30a391ffd10b55d264d3 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 20:30:03 +0000 Subject: [PATCH 05/16] fix: printing with ParetoSingle --- src/HallOfFame.jl | 19 +++++++++++++------ test/test_pretty_printing.jl | 16 ++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 3620db6f1..db958cf46 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -44,6 +44,13 @@ Base.iterate(::ParetoSingle, ::Nothing) = nothing Base.iterate(el::ParetoNeighborhood) = iterate(el.members) Base.iterate(el::ParetoNeighborhood, state) = iterate(el.members, state) +function Base.show(io::IO, mime::MIME"text/plain", el::ParetoSingle) + print(io, "ParetoSingle(") + show(io, mime, el.member) + print(io, ")") + return nothing +end + function _depwarn_pareto_single(funcsym::Symbol) return Base.depwarn( "Hall of fame `.members` is now a vector of `AbstractParetoElement` objects. ", @@ -97,17 +104,17 @@ function Base.getproperty(hof::HallOfFame, name::Symbol) end function Base.show(io::IO, mime::MIME"text/plain", hof::HallOfFame{T,L,N}) where {T,L,N} println(io, "HallOfFame{...}:") - for i in eachindex(hof.members, hof.exists) - s_member, s_exists = if hof.exists[i] - sprint((io, m) -> show(io, mime, m), hof.members[i]), "true" + for i in eachindex(hof.elements, hof.exists) + s_element, s_exists = if hof.exists[i] + sprint((io, m) -> show(io, mime, m), hof.elements[i]), "true" else "undef", "false" end println(io, " "^4 * ".exists[$i] = $s_exists") - print(io, " "^4 * ".members[$i] =") - splitted = split(strip(s_member), '\n') + print(io, " "^4 * ".elements[$i] =") + splitted = split(strip(s_element), '\n') if length(splitted) == 1 - println(io, " " * s_member) + println(io, " " * s_element) else println(io) foreach(line -> println(io, " "^8 * line), splitted) diff --git a/test/test_pretty_printing.jl b/test/test_pretty_printing.jl index dcbc3f59c..299445262 100644 --- a/test/test_pretty_printing.jl +++ b/test/test_pretty_printing.jl @@ -46,24 +46,24 @@ end hof = HallOfFame(options, dataset) hof = embed_metadata(hof, options, dataset) - hof.members[5] = member + hof.elements[5] = ParetoSingle(member) hof.exists[5] = true s_hof = strip(shower(hof)) true_s = "HallOfFame{...}: .exists[1] = false - .members[1] = undef + .elements[1] = undef .exists[2] = false - .members[2] = undef + .elements[2] = undef .exists[3] = false - .members[3] = undef + .elements[3] = undef .exists[4] = false - .members[4] = undef + .elements[4] = undef .exists[5] = true - .members[5] = PopMember(tree = ((x ^ 2.0) + 1.5), loss = 16.25, score = 1.0) + .elements[5] = ParetoSingle(PopMember(tree = ((x ^ 2.0) + 1.5), loss = 16.25, score = 1.0)) .exists[6] = false - .members[6] = undef + .elements[6] = undef .exists[7] = false - .members[7] = undef" + .elements[7] = undef" @test s_hof == true_s end From d79bcfc74bc5866832b1752d830f75d9d26cc13d Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 20:30:29 +0000 Subject: [PATCH 06/16] refactor: clean up some API calls --- src/HallOfFame.jl | 13 ++++++++----- src/SymbolicRegression.jl | 16 +++++++++++++--- test/test_pretty_printing.jl | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index db958cf46..29d7c4a14 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -146,16 +146,19 @@ function HallOfFame( ) return HallOfFame( - [init_element(options.pareto_element_options, member) for i in 1:(options.maxsize)], + [ + init_pareto_element(options.pareto_element_options, member) for + i in 1:(options.maxsize) + ], [false for i in 1:(options.maxsize)], ) end Base.copy(hof::HallOfFame) = HallOfFame(map(copy, hof.members), copy(hof.exists)) -function init_element(::Union{ParetoSingleOptions,ParetoSingle}, member::PopMember) +function init_pareto_element(::Union{ParetoSingleOptions,ParetoSingle}, member::PopMember) return ParetoSingle(copy(member)) end -function init_element( +function init_pareto_element( opt::Union{ParetoNeighborhoodOptions,ParetoNeighborhood}, member::PopMember ) return ParetoNeighborhood([copy(member)], opt.bucket_size) @@ -165,7 +168,7 @@ function Base.push!(hof::HallOfFame, (size, member)::Pair{<:Integer,<:PopMember} maxsize = length(hof.elements) if 0 < size <= maxsize if !hof.exists[size] - hof.elements[size] = init_element(hof.elements[size], member) + hof.elements[size] = init_pareto_element(hof.elements[size], member) hof.exists[size] = true else hof.elements[size] = push!(hof.elements[size], member.score => member) @@ -181,7 +184,7 @@ function Base.push!(el::ParetoNeighborhood, (score, member)::Pair{<:LOSS_TYPE,<: return error("Not implemented") end -function Base.push!(hof::HallOfFame, pop::Population; options::AbstractOptions) +function Base.append!(hof::HallOfFame, pop::Population; options::AbstractOptions) for member in pop.members size = compute_complexity(member, options) push!(hof, size => member) diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index e6d305c4c..541755860 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -285,7 +285,12 @@ using .LossFunctionsModule: eval_loss, score_func, update_baseline_loss! using .PopMemberModule: PopMember, reset_birth! using .PopulationModule: Population, best_sub_pop, record_population, best_of_sample using .HallOfFameModule: - HallOfFame, calculate_pareto_frontier, string_dominating_pareto_curve + HallOfFame, + ParetoSingle, + ParetoNeighborhood, + calculate_pareto_frontier, + string_dominating_pareto_curve, + init_pareto_element using .MutateModule: mutate!, condition_mutation_weights!, MutationResult using .SingleIterationModule: s_r_cycle, optimize_and_simplify_population using .ProgressBarsModule: WrappedProgressBar @@ -595,7 +600,12 @@ end example_ex = create_expression(zero(T), options, example_dataset) NT = typeof(example_ex) PopType = Population{T,L,NT} - HallOfFameType = HallOfFame{T,L,NT} + example_member = PopMember(example_ex, zero(L), zero(L); options.deterministic) + example_pareto_element = init_pareto_element( + options.pareto_element_options, example_member + ) + ParetoElementType = typeof(example_pareto_element) + HallOfFameType = HallOfFame{T,L,NT,ParetoElementType} WorkerOutputType = get_worker_output_type( Val(ropt.parallelism), PopType, HallOfFameType ) @@ -893,7 +903,7 @@ function _main_search_loop!( update_frequencies!(state.all_running_search_statistics[j]; size) end #! format: off - push!(state.halls_of_fame[j], cur_pop; options) + append!(state.halls_of_fame[j], cur_pop; options) merge!(state.halls_of_fame[j], best_seen) #! format: on # TODO: Confirm that `best_seen.members` have full-batch scores diff --git a/test/test_pretty_printing.jl b/test/test_pretty_printing.jl index 299445262..86a3b260f 100644 --- a/test/test_pretty_printing.jl +++ b/test/test_pretty_printing.jl @@ -25,7 +25,7 @@ end @testitem "pretty print hall of fame" tags = [:part1] begin using SymbolicRegression - using SymbolicRegression: embed_metadata + using SymbolicRegression: embed_metadata, ParetoSingle using SymbolicRegression.CoreModule: safe_pow options = Options(; binary_operators=[+, safe_pow], maxsize=7) From 3ecc0bee9bf0e1a8a27801789547915a23433f51 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 20:48:32 +0000 Subject: [PATCH 07/16] fix: type instability in search state --- src/HallOfFame.jl | 4 ++-- src/SearchUtils.jl | 29 +++++++++++++++++++++-------- src/SymbolicRegression.jl | 26 ++++++++++---------------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 29d7c4a14..52930313f 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -53,7 +53,7 @@ end function _depwarn_pareto_single(funcsym::Symbol) return Base.depwarn( - "Hall of fame `.members` is now a vector of `AbstractParetoElement` objects. ", + "Hall of fame `.members` is now `.elements` which is a vector of `AbstractParetoElement` objects. ", funcsym, ) end @@ -153,7 +153,7 @@ function HallOfFame( [false for i in 1:(options.maxsize)], ) end -Base.copy(hof::HallOfFame) = HallOfFame(map(copy, hof.members), copy(hof.exists)) +Base.copy(hof::HallOfFame) = HallOfFame(map(copy, hof.elements), copy(hof.exists)) function init_pareto_element(::Union{ParetoSingleOptions,ParetoSingle}, member::PopMember) return ParetoSingle(copy(member)) diff --git a/src/SearchUtils.jl b/src/SearchUtils.jl index fa9effc41..ba9381736 100644 --- a/src/SearchUtils.jl +++ b/src/SearchUtils.jl @@ -13,11 +13,12 @@ using Logging: AbstractLogger using DynamicExpressions: AbstractExpression, string_tree using ..UtilsModule: subscriptify -using ..CoreModule: Dataset, AbstractOptions, Options, RecordType, max_features +using ..CoreModule: + Dataset, AbstractOptions, AbstractParetoOptions, Options, RecordType, max_features using ..ComplexityModule: compute_complexity using ..PopulationModule: Population using ..PopMemberModule: PopMember -using ..HallOfFameModule: HallOfFame, string_dominating_pareto_curve +using ..HallOfFameModule: HallOfFame, string_dominating_pareto_curve, init_pareto_element using ..ProgressBarsModule: WrappedProgressBar, manually_iterate!, barlen using ..AdaptiveParsimonyModule: RunningSearchStatistics @@ -231,6 +232,17 @@ end const DefaultWorkerOutputType{P,H} = Tuple{P,H,RecordType,Float64} +function get_hall_of_fame_type( + ::Type{T}, + ::Type{L}, + example_ex::AbstractExpression, + pareto_element_options::AbstractParetoOptions, +) where {T,L} + example_member = PopMember(example_ex, zero(L), zero(L); deterministic=false) + example_pareto_element = init_pareto_element(pareto_element_options, example_member) + ParetoElementType = typeof(example_pareto_element) + return HallOfFame{T,L,typeof(example_ex),ParetoElementType} +end function get_worker_output_type( ::Val{PARALLELISM}, ::Type{PopType}, ::Type{HallOfFameType} ) where {PARALLELISM,PopType,HallOfFameType} @@ -518,7 +530,7 @@ end load_saved_population(::Nothing; kws...) = nothing """ - AbstractSearchState{T,L,N} + AbstractSearchState{T,L,N,H} An abstract type encapsulating the internal state of the search process during symbolic regression. @@ -535,17 +547,18 @@ Look through the source of `equation_search` to see how this is used. - [`AbstractOptions`](@ref SymbolicRegression.CoreModule.OptionsStruct.AbstractOptions): See how to extend abstract types for customizing options. """ -abstract type AbstractSearchState{T,L,N<:AbstractExpression{T}} end +abstract type AbstractSearchState{T,L,N<:AbstractExpression{T},H<:HallOfFame{T,L,N}} end """ - SearchState{T,L,N,WorkerOutputType,ChannelType} <: AbstractSearchState{T,L,N} + SearchState{T,L,N,H,WorkerOutputType,ChannelType} <: AbstractSearchState{T,L,N,H} The state of the search, including the populations, worker outputs, tasks, and channels. This is used to manage the search and keep track of runtime variables in a single struct. """ -Base.@kwdef struct SearchState{T,L,N<:AbstractExpression{T},WorkerOutputType,ChannelType} <: - AbstractSearchState{T,L,N} +Base.@kwdef struct SearchState{ + T,L,N<:AbstractExpression{T},H<:HallOfFame{T,L,N},WorkerOutputType,ChannelType +} <: AbstractSearchState{T,L,N,H} procs::Vector{Int} we_created_procs::Bool worker_output::Vector{Vector{WorkerOutputType}} @@ -553,7 +566,7 @@ Base.@kwdef struct SearchState{T,L,N<:AbstractExpression{T},WorkerOutputType,Cha channels::Vector{Vector{ChannelType}} worker_assignment::WorkerAssignments task_order::Vector{Tuple{Int,Int}} - halls_of_fame::Vector{HallOfFame{T,L,N}} + halls_of_fame::Vector{H} last_pops::Vector{Vector{Population{T,L,N}}} best_sub_pops::Vector{Vector{Population{T,L,N}}} all_running_search_statistics::Vector{RunningSearchStatistics} diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index 541755860..fa1175035 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -304,6 +304,7 @@ using .SearchUtilsModule: WorkerAssignments, DefaultWorkerOutputType, assign_next_worker!, + get_hall_of_fame_type, get_worker_output_type, extract_from_worker, @sr_spawner, @@ -598,14 +599,9 @@ end nout = length(datasets) example_dataset = first(datasets) example_ex = create_expression(zero(T), options, example_dataset) - NT = typeof(example_ex) - PopType = Population{T,L,NT} - example_member = PopMember(example_ex, zero(L), zero(L); options.deterministic) - example_pareto_element = init_pareto_element( - options.pareto_element_options, example_member - ) - ParetoElementType = typeof(example_pareto_element) - HallOfFameType = HallOfFame{T,L,NT,ParetoElementType} + ExpressionType = typeof(example_ex) + PopType = Population{T,L,ExpressionType} + HallOfFameType = get_hall_of_fame_type(T, L, example_ex, options.pareto_element_options) WorkerOutputType = get_worker_output_type( Val(ropt.parallelism), PopType, HallOfFameType ) @@ -662,7 +658,7 @@ end j in 1:nout ] - return SearchState{T,L,typeof(example_ex),WorkerOutputType,ChannelType}(; + return SearchState{T,L,typeof(example_ex),HallOfFameType,WorkerOutputType,ChannelType}(; procs=procs, we_created_procs=we_created_procs, worker_output=worker_output, @@ -682,12 +678,12 @@ end ) end function _initialize_search!( - state::AbstractSearchState{T,L,N}, + state::AbstractSearchState{T,L,N,H}, datasets, ropt::AbstractRuntimeOptions, options::AbstractOptions, saved_state, -) where {T,L,N} +) where {T,L,N,H} nout = length(datasets) init_hall_of_fame = load_saved_hall_of_fame(saved_state) @@ -768,11 +764,11 @@ function _initialize_search!( return nothing end function _warmup_search!( - state::AbstractSearchState{T,L,N}, + state::AbstractSearchState{T,L,N,H}, datasets, ropt::AbstractRuntimeOptions, options::AbstractOptions, -) where {T,L,N} +) where {T,L,N,H} nout = length(datasets) for j in 1:nout, i in 1:(options.populations) dataset = datasets[j] @@ -789,9 +785,7 @@ function _warmup_search!( last_pop = state.worker_output[j][i] updated_pop = @sr_spawner( begin - in_pop = first( - extract_from_worker(last_pop, Population{T,L,N}, HallOfFame{T,L,N}) - ) + in_pop = first(extract_from_worker(last_pop, Population{T,L,N}, H)) _dispatch_s_r_cycle( in_pop, dataset, From 0e3e66f3f23d8263d4831533ec53efa04050f284 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 20:52:02 +0000 Subject: [PATCH 08/16] fix: type instability from depwarn --- src/HallOfFame.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 52930313f..76ae03367 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -52,10 +52,11 @@ function Base.show(io::IO, mime::MIME"text/plain", el::ParetoSingle) end function _depwarn_pareto_single(funcsym::Symbol) - return Base.depwarn( + Base.depwarn( "Hall of fame `.members` is now `.elements` which is a vector of `AbstractParetoElement` objects. ", funcsym, ) + return nothing end function Base.getproperty(s::ParetoSingle, name::Symbol) From 1804cf3453fd66c0a8e6258ccd914dec90622b4b Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 20:57:54 +0000 Subject: [PATCH 09/16] fix: other misuses for new element type --- src/SearchUtils.jl | 6 ++++-- src/SymbolicRegression.jl | 8 ++++---- test/test_custom_operators_multiprocessing.jl | 4 ++-- test/test_early_stop.jl | 3 ++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/SearchUtils.jl b/src/SearchUtils.jl index ba9381736..e5a1cdd25 100644 --- a/src/SearchUtils.jl +++ b/src/SearchUtils.jl @@ -368,8 +368,10 @@ function _check_for_loss_threshold(_, ::Nothing, ::AbstractOptions) end function _check_for_loss_threshold(halls_of_fame, f::F, options::AbstractOptions) where {F} return all(halls_of_fame) do hof - any(hof.members[hof.exists]) do member - f(member.loss, compute_complexity(member, options))::Bool + any(hof.elements[hof.exists]) do element + any(element) do member + f(member.loss, compute_complexity(member, options))::Bool + end end end end diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index fa1175035..6e383602e 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -432,10 +432,10 @@ which is useful for debugging and profiling. # Returns - `hallOfFame::HallOfFame`: The best equations seen during the search. - hallOfFame.members gives an array of `PopMember` objects, which - have their tree (equation) stored in `.tree`. Their score (loss) - is given in `.score`. The array of `PopMember` objects - is enumerated by size from `1` to `options.maxsize`. + map(first, hallOfFame.elements) gives an array of `PopMember` objects + These have their tree (equation) stored in `.tree`. Their score (loss) + is given in `.score`. The array is enumerated by size from `1` + to `options.maxsize`. """ function equation_search( X::AbstractMatrix{T}, diff --git a/test/test_custom_operators_multiprocessing.jl b/test/test_custom_operators_multiprocessing.jl index 22c978771..9b3eb3726 100644 --- a/test/test_custom_operators_multiprocessing.jl +++ b/test/test_custom_operators_multiprocessing.jl @@ -48,6 +48,6 @@ hof = equation_search( ) @test any( - early_stop(member.loss, my_complexity(member.tree)) for - member in hof.members[hof.exists] + early_stop(member.loss, my_complexity(member.tree)) for el in hof.elements[hof.exists], + member in el ) diff --git a/test/test_early_stop.jl b/test/test_early_stop.jl index 3ba36e555..f203402df 100644 --- a/test/test_early_stop.jl +++ b/test/test_early_stop.jl @@ -15,5 +15,6 @@ options = SymbolicRegression.Options(; hof = equation_search(X, y; options=options, niterations=1_000_000_000) @test any( - early_stop(member.loss, count_nodes(member.tree)) for member in hof.members[hof.exists] + early_stop(member.loss, count_nodes(member.tree)) for el in hof.elements[hof.exists], + member in el ) From 0e4fb7c13671b0257e9cd33193fb82093b54943d Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 21:12:41 +0000 Subject: [PATCH 10/16] fix: generalize other instances of `.members` to pareto elements --- src/SymbolicRegression.jl | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index 6e383602e..c27754dbd 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -709,6 +709,9 @@ function _initialize_search!( # frontier, we push them individually: push!(new_hof, size => member) end + # TODO: Would be nice if there was a way to mark `init_hall_of_fame[j]` + # as being dead now. We want to make sure we never use it at this point + # in the code. state.halls_of_fame[j] = new_hof end end @@ -1116,15 +1119,23 @@ end dataset, out_pop, options, cur_maxsize, record ) num_evals += evals_from_optimize - if options.batching - for i_member in 1:(options.maxsize) - score, result_loss = score_func(dataset, best_seen.members[i_member], options) - best_seen.members[i_member].score = score - best_seen.members[i_member].loss = result_loss + return_hof = if options.batching + # Compute full-dataset scores for all members of the Pareto front. + new_hof = HallOfFame(options, dataset)::typeof(best_seen) + for el in best_seen.elements[best_seen.exists], member in el + size = compute_complexity(member, options) + score, result_loss = score_func(dataset, member, options) + member.score = score + member.loss = result_loss num_evals += 1 + + push!(new_hof, size => member) end + new_hof + else + best_seen end - return (out_pop, best_seen, record, num_evals) + return (out_pop, return_hof, record, num_evals) end function _info_dump( state::AbstractSearchState, From 8f024d08e099aa2488ac7368aefd5f07b3049a3e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 21:25:44 +0000 Subject: [PATCH 11/16] fix: force inline of ParetoSingle getproperty --- src/HallOfFame.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 76ae03367..8801d48fa 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -59,12 +59,12 @@ function _depwarn_pareto_single(funcsym::Symbol) return nothing end -function Base.getproperty(s::ParetoSingle, name::Symbol) +@inline function Base.getproperty(s::ParetoSingle, name::Symbol) name == :member && return getfield(s, :member) _depwarn_pareto_single(:getproperty) return getproperty(s.member, name) end -function Base.setproperty!(s::ParetoSingle, name::Symbol, value) +@inline function Base.setproperty!(s::ParetoSingle, name::Symbol, value) name == :member && return setfield!(s, :member, value) _depwarn_pareto_single(:setproperty!) return setproperty!(s.member, name, value) @@ -93,7 +93,7 @@ struct HallOfFame{ exists::Vector{Bool} end pop_member_type(::Type{<:HallOfFame{T,L,N,H}}) where {T,L,N,H} = pop_member_type(H) -function Base.getproperty(hof::HallOfFame, name::Symbol) +@inline function Base.getproperty(hof::HallOfFame, name::Symbol) if name == :members Base.depwarn( "HallOfFame.members is deprecated. Use HallOfFame.elements instead.", From aedcc5f258df2d4e48f0b61af0a66f6d2e3c892f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 22:04:43 +0000 Subject: [PATCH 12/16] feat: implement utility functions for ParetoNeighborhood --- src/HallOfFame.jl | 48 +++++++++++++++++++++++++++++++++++++++++--- src/OptionsStruct.jl | 2 +- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 8801d48fa..e45ce1e40 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -162,7 +162,9 @@ end function init_pareto_element( opt::Union{ParetoNeighborhoodOptions,ParetoNeighborhood}, member::PopMember ) - return ParetoNeighborhood([copy(member)], opt.bucket_size) + members = sizehint!(typeof(member)[], opt.bucket_size + 1) + push!(members, copy(member)) + return ParetoNeighborhood(members, opt.bucket_size) end function Base.push!(hof::HallOfFame, (size, member)::Pair{<:Integer,<:PopMember}) @@ -182,7 +184,26 @@ function Base.push!(el::ParetoSingle, (score, member)::Pair{<:LOSS_TYPE,<:PopMem return el.member.score > score ? ParetoSingle(copy(member)) : el end function Base.push!(el::ParetoNeighborhood, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) - return error("Not implemented") + if isempty(el.members) + push!(el.members, copy(member)) + return el + elseif el.members[end].score <= score + # No update needed + return el + elseif el.members[1].score > score + pushfirst!(el.members, copy(member)) + else + # Find the first member with worse score + i = findfirst(m -> m.score > score, el.members)::Int + # member assumes that position, and pushes the array forward + insert!(el.members, i, copy(member)) + end + + if length(el.members) > el.bucket_size + pop!(el.members) + end + + return el end function Base.append!(hof::HallOfFame, pop::Population; options::AbstractOptions) @@ -211,7 +232,28 @@ function Base.merge(el1::ParetoSingle, el2::ParetoSingle) return el1.member.score <= el2.member.score ? el1 : copy(el2) end function Base.merge(el1::ParetoNeighborhood, el2::ParetoNeighborhood) - return error("Not implemented") + P = pop_member_type(typeof(el1)) + new_neighborhood = sizehint!(P[], el1.bucket_size + 1) + i1 = firstindex(el1.members) + n1 = length(el1.members) + i2 = firstindex(el2.members) + n2 = length(el2.members) + i = 1 + while i1 <= n1 && i2 <= n2 && i <= el1.bucket_size + m1 = el1.members[i1] + m2 = el2.members[i2] + if m1.score <= m2.score + # TODO: Is it safe that we don't copy here? I think so; since we are merging + # onto el1 (see `Base.merge!`), but perhaps someone could misuse this. + push!(new_neighborhood, m1) + i1 += 1 + else + push!(new_neighborhood, copy(m2)) + i2 += 1 + end + i += 1 + end + return ParetoNeighborhood(new_neighborhood, el1.bucket_size) end """ diff --git a/src/OptionsStruct.jl b/src/OptionsStruct.jl index d2bd777de..aa807dfe4 100644 --- a/src/OptionsStruct.jl +++ b/src/OptionsStruct.jl @@ -20,7 +20,7 @@ Abstract type for different Pareto front storage strategies. abstract type AbstractParetoOptions end struct ParetoSingleOptions <: AbstractParetoOptions end -struct ParetoNeighborhoodOptions <: AbstractParetoOptions +Base.@kwdef struct ParetoNeighborhoodOptions <: AbstractParetoOptions bucket_size::Int end From a1036e60d0405fcae952314f1552feda78fff392 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 22:07:13 +0000 Subject: [PATCH 13/16] refactor: rename `ParetoNeighborhood` to `ParetoTopK` --- src/Core.jl | 2 +- src/ExpressionBuilder.jl | 15 +++++++-------- src/HallOfFame.jl | 37 +++++++++++++++++-------------------- src/OptionsStruct.jl | 6 +++--- src/SymbolicRegression.jl | 4 ++-- 5 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/Core.jl b/src/Core.jl index 62d2e7732..2fc9a55cb 100644 --- a/src/Core.jl +++ b/src/Core.jl @@ -19,7 +19,7 @@ using .OptionsStructModule: ComplexityMapping, AbstractParetoOptions, ParetoSingleOptions, - ParetoNeighborhoodOptions, + ParetoTopKOptions, specialized_options, operator_specialization using .OperatorsModule: diff --git a/src/ExpressionBuilder.jl b/src/ExpressionBuilder.jl index 34479835e..e22bebd6d 100644 --- a/src/ExpressionBuilder.jl +++ b/src/ExpressionBuilder.jl @@ -10,7 +10,7 @@ using DynamicExpressions: AbstractExpressionNode, AbstractExpression, constructorof, with_metadata using StatsBase: StatsBase using ..CoreModule: AbstractOptions, Dataset -using ..HallOfFameModule: HallOfFame, ParetoSingle, ParetoNeighborhood +using ..HallOfFameModule: HallOfFame, ParetoSingle, ParetoTopK using ..PopulationModule: Population using ..PopMemberModule: PopMember @@ -133,11 +133,10 @@ end return ParetoSingle(embed_metadata(el.member, options, dataset)) end function embed_metadata( - el::ParetoNeighborhood, options::AbstractOptions, dataset::Dataset{T,L} + el::ParetoTopK, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L} - return ParetoNeighborhood( - map(member -> embed_metadata(member, options, dataset), el.members), - el.bucket_size, + return ParetoTopK( + map(member -> embed_metadata(member, options, dataset), el.members), el.k ) end function embed_metadata( @@ -190,10 +189,10 @@ function strip_metadata( return ParetoSingle(strip_metadata(el.member, options, dataset)) end function strip_metadata( - el::ParetoNeighborhood, options::AbstractOptions, dataset::Dataset{T,L} + el::ParetoTopK, options::AbstractOptions, dataset::Dataset{T,L} ) where {T,L} - return ParetoNeighborhood( - map(member -> strip_metadata(member, options, dataset), el.members), el.bucket_size + return ParetoTopK( + map(member -> strip_metadata(member, options, dataset), el.members), el.k ) end function strip_metadata( diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index e45ce1e40..ee50d6add 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -2,10 +2,9 @@ module HallOfFameModule using StyledStrings: @styled_str using DynamicExpressions: AbstractExpression, string_tree -# using DataStructures: PriorityQueue using Printf: @sprintf using ..UtilsModule: split_string, AnnotatedIOBuffer, dump_buffer -using ..CoreModule: ParetoSingleOptions, ParetoNeighborhoodOptions +using ..CoreModule: ParetoSingleOptions, ParetoTopKOptions using ..CoreModule: AbstractOptions, Dataset, DATA_TYPE, LOSS_TYPE, relu, create_expression using ..ComplexityModule: compute_complexity using ..PopMemberModule: PopMember @@ -19,7 +18,7 @@ Abstract type for storing elements on the Pareto frontier. # Subtypes - `ParetoSingle`: Stores a single member at each complexity level -- `ParetoNeighborhood`: Stores multiple members at each complexity level in a fixed-size bucket +- `ParetoTopK`: Stores multiple members at each complexity level in a fixed-size bucket """ abstract type AbstractParetoElement{P<:PopMember} end @@ -28,21 +27,21 @@ pop_member_type(::Type{<:AbstractParetoElement{P}}) where {P} = P struct ParetoSingle{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} member::P end -struct ParetoNeighborhood{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} +struct ParetoTopK{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} members::Vector{P} - bucket_size::Int + k::Int end Base.copy(el::ParetoSingle) = ParetoSingle(copy(el.member)) -Base.copy(el::ParetoNeighborhood) = ParetoNeighborhood(copy(el.members), el.bucket_size) +Base.copy(el::ParetoTopK) = ParetoTopK(copy(el.members), el.k) Base.first(el::ParetoSingle) = el.member -Base.first(el::ParetoNeighborhood) = first(el.members) +Base.first(el::ParetoTopK) = first(el.members) Base.iterate(el::ParetoSingle) = (el.member, nothing) Base.iterate(::ParetoSingle, ::Nothing) = nothing -Base.iterate(el::ParetoNeighborhood) = iterate(el.members) -Base.iterate(el::ParetoNeighborhood, state) = iterate(el.members, state) +Base.iterate(el::ParetoTopK) = iterate(el.members) +Base.iterate(el::ParetoTopK, state) = iterate(el.members, state) function Base.show(io::IO, mime::MIME"text/plain", el::ParetoSingle) print(io, "ParetoSingle(") @@ -159,12 +158,10 @@ Base.copy(hof::HallOfFame) = HallOfFame(map(copy, hof.elements), copy(hof.exists function init_pareto_element(::Union{ParetoSingleOptions,ParetoSingle}, member::PopMember) return ParetoSingle(copy(member)) end -function init_pareto_element( - opt::Union{ParetoNeighborhoodOptions,ParetoNeighborhood}, member::PopMember -) - members = sizehint!(typeof(member)[], opt.bucket_size + 1) +function init_pareto_element(opt::Union{ParetoTopKOptions,ParetoTopK}, member::PopMember) + members = sizehint!(typeof(member)[], opt.k + 1) push!(members, copy(member)) - return ParetoNeighborhood(members, opt.bucket_size) + return ParetoTopK(members, opt.k) end function Base.push!(hof::HallOfFame, (size, member)::Pair{<:Integer,<:PopMember}) @@ -183,7 +180,7 @@ end function Base.push!(el::ParetoSingle, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) return el.member.score > score ? ParetoSingle(copy(member)) : el end -function Base.push!(el::ParetoNeighborhood, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) +function Base.push!(el::ParetoTopK, (score, member)::Pair{<:LOSS_TYPE,<:PopMember}) if isempty(el.members) push!(el.members, copy(member)) return el @@ -199,7 +196,7 @@ function Base.push!(el::ParetoNeighborhood, (score, member)::Pair{<:LOSS_TYPE,<: insert!(el.members, i, copy(member)) end - if length(el.members) > el.bucket_size + if length(el.members) > el.k pop!(el.members) end @@ -231,15 +228,15 @@ function Base.merge(el1::ParetoSingle, el2::ParetoSingle) # Remember: we want the MIN score (bad API choice, but we're stuck with it for now) return el1.member.score <= el2.member.score ? el1 : copy(el2) end -function Base.merge(el1::ParetoNeighborhood, el2::ParetoNeighborhood) +function Base.merge(el1::ParetoTopK, el2::ParetoTopK) P = pop_member_type(typeof(el1)) - new_neighborhood = sizehint!(P[], el1.bucket_size + 1) + new_neighborhood = sizehint!(P[], el1.k + 1) i1 = firstindex(el1.members) n1 = length(el1.members) i2 = firstindex(el2.members) n2 = length(el2.members) i = 1 - while i1 <= n1 && i2 <= n2 && i <= el1.bucket_size + while i1 <= n1 && i2 <= n2 && i <= el1.k m1 = el1.members[i1] m2 = el2.members[i2] if m1.score <= m2.score @@ -253,7 +250,7 @@ function Base.merge(el1::ParetoNeighborhood, el2::ParetoNeighborhood) end i += 1 end - return ParetoNeighborhood(new_neighborhood, el1.bucket_size) + return ParetoTopK(new_neighborhood, el1.k) end """ diff --git a/src/OptionsStruct.jl b/src/OptionsStruct.jl index aa807dfe4..f9ff86313 100644 --- a/src/OptionsStruct.jl +++ b/src/OptionsStruct.jl @@ -15,13 +15,13 @@ Abstract type for different Pareto front storage strategies. # Subtypes - `ParetoSingleOptions`: Store only the single best member at each complexity -- `ParetoNeighborhoodOptions`: Store multiple members at each complexity in a fixed-size bucket +- `ParetoTopKOptions`: Store multiple members at each complexity in a fixed-size bucket """ abstract type AbstractParetoOptions end struct ParetoSingleOptions <: AbstractParetoOptions end -Base.@kwdef struct ParetoNeighborhoodOptions <: AbstractParetoOptions - bucket_size::Int +Base.@kwdef struct ParetoTopKOptions <: AbstractParetoOptions + k::Int end """ diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index c27754dbd..db1709821 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -236,7 +236,7 @@ using .CoreModule: MutationWeights, AbstractParetoOptions, ParetoSingleOptions, - ParetoNeighborhoodOptions, + ParetoTopKOptions, get_safe_op, max_features, is_weighted, @@ -287,7 +287,7 @@ using .PopulationModule: Population, best_sub_pop, record_population, best_of_sa using .HallOfFameModule: HallOfFame, ParetoSingle, - ParetoNeighborhood, + ParetoTopK, calculate_pareto_frontier, string_dominating_pareto_curve, init_pareto_element From f191d3807f8f4df2e4ac4b79f7bcb948ad2d0004 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 22:08:33 +0000 Subject: [PATCH 14/16] feat: more `sizehint!` for pareto elements --- src/HallOfFame.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index ee50d6add..7ee6e4fef 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -33,7 +33,7 @@ struct ParetoTopK{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} end Base.copy(el::ParetoSingle) = ParetoSingle(copy(el.member)) -Base.copy(el::ParetoTopK) = ParetoTopK(copy(el.members), el.k) +Base.copy(el::ParetoTopK) = ParetoTopK(sizehint!(copy(el.members), el.k + 1), el.k) Base.first(el::ParetoSingle) = el.member Base.first(el::ParetoTopK) = first(el.members) From 78261a7dac95df2e9f477ee15a36a952d7c3d896 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 22:15:37 +0000 Subject: [PATCH 15/16] fix: aliasing issue from shallow copy --- src/HallOfFame.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HallOfFame.jl b/src/HallOfFame.jl index 7ee6e4fef..dd3f6f887 100644 --- a/src/HallOfFame.jl +++ b/src/HallOfFame.jl @@ -33,7 +33,7 @@ struct ParetoTopK{T,L,N,P<:PopMember{T,L,N}} <: AbstractParetoElement{P} end Base.copy(el::ParetoSingle) = ParetoSingle(copy(el.member)) -Base.copy(el::ParetoTopK) = ParetoTopK(sizehint!(copy(el.members), el.k + 1), el.k) +Base.copy(el::ParetoTopK) = ParetoTopK(sizehint!(map(copy, el.members), el.k + 1), el.k) Base.first(el::ParetoSingle) = el.member Base.first(el::ParetoTopK) = first(el.members) From dcc0bae2f311c3e669b8c01f14bf15163c8f0820 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Sat, 4 Jan 2025 22:58:13 +0000 Subject: [PATCH 16/16] feat: migrate entire hof rather than only dominating --- src/SymbolicRegression.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/SymbolicRegression.jl b/src/SymbolicRegression.jl index db1709821..0845efcca 100644 --- a/src/SymbolicRegression.jl +++ b/src/SymbolicRegression.jl @@ -922,7 +922,12 @@ function _main_search_loop!( ) end if options.hof_migration && length(dominating) > 0 - migrate!(dominating => cur_pop, options; frac=options.fraction_replaced_hof) + all_hof_members = [ + member for el in state.halls_of_fame[j].elements for member in el + ] + migrate!( + all_hof_members => cur_pop, options; frac=options.fraction_replaced_hof + ) end ###################################################################