From 673325d73b46ef4d487c9fe971108c4b20a6b63c Mon Sep 17 00:00:00 2001 From: aaccensi Date: Mon, 11 May 2026 17:23:17 +0200 Subject: [PATCH 1/3] Batch etsource cache reads with fetch_multi to reduce N+1 queries MeritOrder collapses 6 individual cache fetches into one fetch_multi call. Molecules does the same for its 2 keys. Both use Thread.current to ensure fetch_multi runs at most once per request. Closes #1747 --- app/models/etsource/merit_order.rb | 45 ++++++++++++++++++++++-------- app/models/etsource/molecules.rb | 22 +++++++++++---- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/app/models/etsource/merit_order.rb b/app/models/etsource/merit_order.rb index 52e69916d..cb96aa110 100644 --- a/app/models/etsource/merit_order.rb +++ b/app/models/etsource/merit_order.rb @@ -1,5 +1,14 @@ module Etsource class MeritOrder + CACHE_ATTRIBUTES = %i[ + merit_order + hydrogen + heat_network_lt + heat_network_mt + heat_network_ht + agriculture_heat + ].freeze + def initialize(etsource = Etsource::Base.instance) @etsource = etsource end @@ -84,21 +93,35 @@ def import_agriculture_heat # # Returns a hash. def import(attribute) - Rails.cache.fetch("#{attribute}_hash") do - mo_nodes = Atlas::EnergyNode.all.select(&attribute).sort_by(&:key) - mo_data = {} + merit_order_cache[attribute] + end - mo_nodes.each do |node| - config = node.public_send(attribute) - type = config.type.to_s - group = config.group&.to_s + # Batches all MeritOrder cache reads into one SQL query per request, so + # subsequent fetch calls hit the LocalStore instead of the database. + def merit_order_cache + Thread.current[:merit_order_cache] ||= begin + cache_attr_by_key = CACHE_ATTRIBUTES.index_by { |a| "#{a}_hash" } - mo_data[type] ||= {} - mo_data[type][node.key.to_s] = group - end + Rails.cache.fetch_multi(*cache_attr_by_key.keys) do |cache_key| + compute(cache_attr_by_key[cache_key]) + end.transform_keys { |k| cache_attr_by_key[k] } + end + end - mo_data + def compute(attribute) + mo_nodes = Atlas::EnergyNode.all.select(&attribute).sort_by(&:key) + mo_data = {} + + mo_nodes.each do |node| + config = node.public_send(attribute) + type = config.type.to_s + group = config.group&.to_s + + mo_data[type] ||= {} + mo_data[type][node.key.to_s] = group end + + mo_data end end end diff --git a/app/models/etsource/molecules.rb b/app/models/etsource/molecules.rb index 8db8dc181..b7795f708 100644 --- a/app/models/etsource/molecules.rb +++ b/app/models/etsource/molecules.rb @@ -3,6 +3,11 @@ module Etsource # Loads data relating to the calculation of molecule flows based on the energy graph. module Molecules + CACHE_KEYS = %w[ + molecules.from_energy_keys + molecules.from_molecules_keys + ].freeze + module_function # Internal: Computes the list of molecule graph nodes which have a molecule_conversion @@ -12,9 +17,7 @@ module Molecules # # Returns an Array of Symbols. def from_energy_keys - Rails.cache.fetch('molecules.from_energy_keys') do - Atlas::MoleculeNode.all.select(&:from_energy).map(&:key).sort - end + molecule_cache['molecules.from_energy_keys'] end # Internal: Computes the list of molecule graph nodes which have a molecule_conversion @@ -24,8 +27,17 @@ def from_energy_keys # # Returns an Array of Symbols. def from_molecules_keys - Rails.cache.fetch('molecules.from_molecules_keys') do - Atlas::EnergyNode.all.select(&:from_molecules).map(&:key).sort + molecule_cache['molecules.from_molecules_keys'] + end + + # Batches all Molecules cache reads into one SQL query per request, so + # subsequent fetch calls hit the LocalStore instead of the database. + def molecule_cache + Thread.current[:molecule_cache] ||= Rails.cache.fetch_multi(*CACHE_KEYS) do |key| + case key + when 'molecules.from_energy_keys' then Atlas::MoleculeNode.all.select(&:from_energy).map(&:key).sort + when 'molecules.from_molecules_keys' then Atlas::EnergyNode.all.select(&:from_molecules).map(&:key).sort + end end end end From 03486271f07d16431590bc567591d740d8afaeb5 Mon Sep 17 00:00:00 2001 From: aaccensi Date: Mon, 18 May 2026 09:51:43 +0200 Subject: [PATCH 2/3] Use process-level memoization instead of IsolatedExecutionState Atlas data only changes on ETSource import, so per-request scoping is wasteful. Class-level memoization is reset via NastyCache#expire_local! whenever an import clears Rails.cache. Closes #1747 --- app/models/etsource/merit_order.rb | 12 ++++++++++-- app/models/etsource/molecules.rb | 8 ++++++-- app/models/nasty_cache.rb | 5 +++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/models/etsource/merit_order.rb b/app/models/etsource/merit_order.rb index cb96aa110..a00a638c2 100644 --- a/app/models/etsource/merit_order.rb +++ b/app/models/etsource/merit_order.rb @@ -1,5 +1,9 @@ module Etsource class MeritOrder + class << self + attr_accessor :cache + end + CACHE_ATTRIBUTES = %i[ merit_order hydrogen @@ -96,10 +100,14 @@ def import(attribute) merit_order_cache[attribute] end + def self.reset_cache! + self.cache = nil + end + # Batches all MeritOrder cache reads into one SQL query per request, so - # subsequent fetch calls hit the LocalStore instead of the database. + # subsequent fetch calls are served from memory instead of the database. def merit_order_cache - Thread.current[:merit_order_cache] ||= begin + self.class.cache ||= begin cache_attr_by_key = CACHE_ATTRIBUTES.index_by { |a| "#{a}_hash" } Rails.cache.fetch_multi(*cache_attr_by_key.keys) do |cache_key| diff --git a/app/models/etsource/molecules.rb b/app/models/etsource/molecules.rb index b7795f708..e43945cd6 100644 --- a/app/models/etsource/molecules.rb +++ b/app/models/etsource/molecules.rb @@ -30,10 +30,14 @@ def from_molecules_keys molecule_cache['molecules.from_molecules_keys'] end + def reset_cache! + @molecule_cache = nil + end + # Batches all Molecules cache reads into one SQL query per request, so - # subsequent fetch calls hit the LocalStore instead of the database. + # subsequent fetch calls are served from memory instead of the database. def molecule_cache - Thread.current[:molecule_cache] ||= Rails.cache.fetch_multi(*CACHE_KEYS) do |key| + @molecule_cache ||= Rails.cache.fetch_multi(*CACHE_KEYS) do |key| case key when 'molecules.from_energy_keys' then Atlas::MoleculeNode.all.select(&:from_energy).map(&:key).sort when 'molecules.from_molecules_keys' then Atlas::EnergyNode.all.select(&:from_molecules).map(&:key).sort diff --git a/app/models/nasty_cache.rb b/app/models/nasty_cache.rb index 8d7088481..b8789b46b 100644 --- a/app/models/nasty_cache.rb +++ b/app/models/nasty_cache.rb @@ -145,6 +145,11 @@ def expire_local! Merit::Curve.reader = Merit::Curve.reader.class.new @cache_store = {} + + # Reset process-level memoization so the next request re-fetches from + # Rails.cache, which has just been cleared by expire_cache! + Etsource::MeritOrder.reset_cache! + Etsource::Molecules.reset_cache! end def expire_atlas!(options) From 1276fcf60170b23b451cac513c54f379920b2ab3 Mon Sep 17 00:00:00 2001 From: aaccensi Date: Tue, 19 May 2026 16:55:47 +0200 Subject: [PATCH 3/3] Use NastyCache instead of Rails.cache for MeritOrder and Molecules NastyCache stores data in process memory, avoiding a cache round-trip on every access, and is automatically cleared on ETSource import. --- app/models/etsource/merit_order.rb | 53 +++++++----------------------- app/models/etsource/molecules.rb | 26 +++------------ app/models/nasty_cache.rb | 5 --- 3 files changed, 16 insertions(+), 68 deletions(-) diff --git a/app/models/etsource/merit_order.rb b/app/models/etsource/merit_order.rb index a00a638c2..171bfb220 100644 --- a/app/models/etsource/merit_order.rb +++ b/app/models/etsource/merit_order.rb @@ -1,18 +1,5 @@ module Etsource class MeritOrder - class << self - attr_accessor :cache - end - - CACHE_ATTRIBUTES = %i[ - merit_order - hydrogen - heat_network_lt - heat_network_mt - heat_network_ht - agriculture_heat - ].freeze - def initialize(etsource = Etsource::Base.instance) @etsource = etsource end @@ -97,39 +84,21 @@ def import_agriculture_heat # # Returns a hash. def import(attribute) - merit_order_cache[attribute] - end + NastyCache.instance.fetch("#{attribute}_hash") do + mo_nodes = Atlas::EnergyNode.all.select(&attribute).sort_by(&:key) + mo_data = {} - def self.reset_cache! - self.cache = nil - end + mo_nodes.each do |node| + config = node.public_send(attribute) + type = config.type.to_s + group = config.group&.to_s - # Batches all MeritOrder cache reads into one SQL query per request, so - # subsequent fetch calls are served from memory instead of the database. - def merit_order_cache - self.class.cache ||= begin - cache_attr_by_key = CACHE_ATTRIBUTES.index_by { |a| "#{a}_hash" } + mo_data[type] ||= {} + mo_data[type][node.key.to_s] = group + end - Rails.cache.fetch_multi(*cache_attr_by_key.keys) do |cache_key| - compute(cache_attr_by_key[cache_key]) - end.transform_keys { |k| cache_attr_by_key[k] } + mo_data end end - - def compute(attribute) - mo_nodes = Atlas::EnergyNode.all.select(&attribute).sort_by(&:key) - mo_data = {} - - mo_nodes.each do |node| - config = node.public_send(attribute) - type = config.type.to_s - group = config.group&.to_s - - mo_data[type] ||= {} - mo_data[type][node.key.to_s] = group - end - - mo_data - end end end diff --git a/app/models/etsource/molecules.rb b/app/models/etsource/molecules.rb index e43945cd6..318fefc65 100644 --- a/app/models/etsource/molecules.rb +++ b/app/models/etsource/molecules.rb @@ -3,11 +3,6 @@ module Etsource # Loads data relating to the calculation of molecule flows based on the energy graph. module Molecules - CACHE_KEYS = %w[ - molecules.from_energy_keys - molecules.from_molecules_keys - ].freeze - module_function # Internal: Computes the list of molecule graph nodes which have a molecule_conversion @@ -17,7 +12,9 @@ module Molecules # # Returns an Array of Symbols. def from_energy_keys - molecule_cache['molecules.from_energy_keys'] + NastyCache.instance.fetch('molecules.from_energy_keys') do + Atlas::MoleculeNode.all.select(&:from_energy).map(&:key).sort + end end # Internal: Computes the list of molecule graph nodes which have a molecule_conversion @@ -27,21 +24,8 @@ def from_energy_keys # # Returns an Array of Symbols. def from_molecules_keys - molecule_cache['molecules.from_molecules_keys'] - end - - def reset_cache! - @molecule_cache = nil - end - - # Batches all Molecules cache reads into one SQL query per request, so - # subsequent fetch calls are served from memory instead of the database. - def molecule_cache - @molecule_cache ||= Rails.cache.fetch_multi(*CACHE_KEYS) do |key| - case key - when 'molecules.from_energy_keys' then Atlas::MoleculeNode.all.select(&:from_energy).map(&:key).sort - when 'molecules.from_molecules_keys' then Atlas::EnergyNode.all.select(&:from_molecules).map(&:key).sort - end + NastyCache.instance.fetch('molecules.from_molecules_keys') do + Atlas::EnergyNode.all.select(&:from_molecules).map(&:key).sort end end end diff --git a/app/models/nasty_cache.rb b/app/models/nasty_cache.rb index b8789b46b..8d7088481 100644 --- a/app/models/nasty_cache.rb +++ b/app/models/nasty_cache.rb @@ -145,11 +145,6 @@ def expire_local! Merit::Curve.reader = Merit::Curve.reader.class.new @cache_store = {} - - # Reset process-level memoization so the next request re-fetches from - # Rails.cache, which has just been cleared by expire_cache! - Etsource::MeritOrder.reset_cache! - Etsource::Molecules.reset_cache! end def expire_atlas!(options)