Skip to content

Commit 8ac2af2

Browse files
fractaledminddhh
fractaledmind
and
dhh
authored
Allow overriding SQLite defaults from database.yml (rails#50460)
* Allow overriding SQLite defaults from `database.yml` Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well. ```yaml database: storage/development.sqlite3 timeout: 5000 pragmas: synchronous: full temp_store: memory ``` * Style * Allow overriding SQLite defaults from `database.yml` Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well. ```yaml database: storage/development.sqlite3 timeout: 5000 pragmas: synchronous: full temp_store: memory ``` --------- Co-authored-by: David Heinemeier Hansson <[email protected]>
1 parent 2edf4a7 commit 8ac2af2

File tree

3 files changed

+281
-24
lines changed

3 files changed

+281
-24
lines changed

‎activerecord/CHANGELOG.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
* Allow overriding SQLite defaults from `database.yml`
2+
3+
Any PRAGMA configuration set under the `pragmas` key in the configuration file take precedence over Rails' defaults, and additional PRAGMAs can be set as well.
4+
5+
```yaml
6+
database: storage/development.sqlite3
7+
timeout: 5000
8+
pragmas:
9+
journal_mode: off
10+
temp_store: memory
11+
```
12+
13+
*Stephen Margheim*
14+
115
* Remove warning message when running SQLite in production, but leave it unconfigured
216
317
There are valid use cases for running SQLite in production, however it must be done

‎activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,15 @@ def dbconsole(config, options = {})
7878
json: { name: "json" },
7979
}
8080

81+
DEFAULT_PRAGMAS = {
82+
"foreign_keys" => true,
83+
"journal_mode" => :wal,
84+
"synchronous" => :normal,
85+
"mmap_size" => 134217728, # 128 megabytes
86+
"journal_size_limit" => 67108864, # 64 megabytes
87+
"cache_size" => 2000
88+
}
89+
8190
class StatementPool < ConnectionAdapters::StatementPool # :nodoc:
8291
alias reset clear
8392

@@ -759,29 +768,14 @@ def configure_connection
759768

760769
super
761770

762-
# Enforce foreign key constraints
763-
# https://www.sqlite.org/pragma.html#pragma_foreign_keys
764-
# https://www.sqlite.org/foreignkeys.html
765-
raw_execute("PRAGMA foreign_keys = ON", "SCHEMA")
766-
unless @memory_database
767-
# Journal mode WAL allows for greater concurrency (many readers + one writer)
768-
# https://www.sqlite.org/pragma.html#pragma_journal_mode
769-
raw_execute("PRAGMA journal_mode = WAL", "SCHEMA")
770-
# Set more relaxed level of database durability
771-
# 2 = "FULL" (sync on every write), 1 = "NORMAL" (sync every 1000 written pages) and 0 = "NONE"
772-
# https://www.sqlite.org/pragma.html#pragma_synchronous
773-
raw_execute("PRAGMA synchronous = NORMAL", "SCHEMA")
774-
# Set the global memory map so all processes can share some data
775-
# https://www.sqlite.org/pragma.html#pragma_mmap_size
776-
# https://www.sqlite.org/mmap.html
777-
raw_execute("PRAGMA mmap_size = #{128.megabytes}", "SCHEMA")
771+
pragmas = @config.fetch(:pragmas, {}).stringify_keys
772+
DEFAULT_PRAGMAS.merge(pragmas).each do |pragma, value|
773+
if ::SQLite3::Pragmas.method_defined?("#{pragma}=")
774+
@raw_connection.public_send("#{pragma}=", value)
775+
else
776+
warn "Unknown SQLite pragma: #{pragma}"
777+
end
778778
end
779-
# Impose a limit on the WAL file to prevent unlimited growth
780-
# https://www.sqlite.org/pragma.html#pragma_journal_size_limit
781-
raw_execute("PRAGMA journal_size_limit = #{64.megabytes}", "SCHEMA")
782-
# Set the local connection cache to 2000 pages
783-
# https://www.sqlite.org/pragma.html#pragma_cache_size
784-
raw_execute("PRAGMA cache_size = 2000", "SCHEMA")
785779
end
786780
end
787781
ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter)

‎activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb

Lines changed: 251 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ def test_default_pragmas
159159
if in_memory_db?
160160
assert_equal [{ "foreign_keys" => 1 }], @conn.execute("PRAGMA foreign_keys")
161161
assert_equal [{ "journal_mode" => "memory" }], @conn.execute("PRAGMA journal_mode")
162-
assert_equal [{ "synchronous" => 2 }], @conn.execute("PRAGMA synchronous")
162+
assert_equal [{ "synchronous" => 1 }], @conn.execute("PRAGMA synchronous")
163163
assert_equal [{ "journal_size_limit" => 67108864 }], @conn.execute("PRAGMA journal_size_limit")
164164
assert_equal [], @conn.execute("PRAGMA mmap_size")
165165
assert_equal [{ "cache_size" => 2000 }], @conn.execute("PRAGMA cache_size")
@@ -175,6 +175,245 @@ def test_default_pragmas
175175
end
176176
end
177177

178+
def test_overriding_default_foreign_keys_pragma
179+
method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
180+
181+
send(method_name, pragmas: { foreign_keys: false }) do |conn|
182+
assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
183+
end
184+
185+
send(method_name, pragmas: { foreign_keys: 0 }) do |conn|
186+
assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
187+
end
188+
189+
send(method_name, pragmas: { foreign_keys: "false" }) do |conn|
190+
assert_equal [{ "foreign_keys" => 0 }], conn.execute("PRAGMA foreign_keys")
191+
end
192+
193+
error = assert_raises(ActiveRecord::StatementInvalid) do
194+
send(method_name, pragmas: { foreign_keys: :false }) do |conn|
195+
conn.execute("PRAGMA foreign_keys")
196+
end
197+
end
198+
assert_match(/unrecognized pragma parameter :false/, error.message)
199+
end
200+
201+
def test_overriding_default_journal_mode_pragma
202+
# in-memory databases are always only ever in `memory` journal_mode
203+
if in_memory_db?
204+
with_memory_connection(pragmas: { "journal_mode" => "delete" }) do |conn|
205+
assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
206+
end
207+
208+
with_memory_connection(pragmas: { "journal_mode" => :delete }) do |conn|
209+
assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
210+
end
211+
212+
error = assert_raises(ActiveRecord::StatementInvalid) do
213+
with_memory_connection(pragmas: { "journal_mode" => 0 }) do |conn|
214+
conn.execute("PRAGMA journal_mode")
215+
end
216+
end
217+
assert_match(/nrecognized journal_mode 0/, error.message)
218+
219+
error = assert_raises(ActiveRecord::StatementInvalid) do
220+
with_memory_connection(pragmas: { "journal_mode" => false }) do |conn|
221+
conn.execute("PRAGMA journal_mode")
222+
end
223+
end
224+
assert_match(/nrecognized journal_mode false/, error.message)
225+
else
226+
# must use a new, separate database file that hasn't been opened in WAL mode before
227+
with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => "delete" }) do |conn|
228+
assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode")
229+
end
230+
231+
with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => :delete }) do |conn|
232+
assert_equal [{ "journal_mode" => "delete" }], conn.execute("PRAGMA journal_mode")
233+
end
234+
235+
error = assert_raises(ActiveRecord::StatementInvalid) do
236+
with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => 0 }) do |conn|
237+
conn.execute("PRAGMA journal_mode")
238+
end
239+
end
240+
assert_match(/unrecognized journal_mode 0/, error.message)
241+
242+
error = assert_raises(ActiveRecord::StatementInvalid) do
243+
with_file_connection(database: "fixtures/journal_mode_test.sqlite3", pragmas: { "journal_mode" => false }) do |conn|
244+
conn.execute("PRAGMA journal_mode")
245+
end
246+
end
247+
assert_match(/unrecognized journal_mode false/, error.message)
248+
end
249+
end
250+
251+
def test_overriding_default_synchronous_pragma
252+
method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
253+
254+
send(method_name, pragmas: { synchronous: :full }) do |conn|
255+
assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
256+
end
257+
258+
send(method_name, pragmas: { synchronous: 2 }) do |conn|
259+
assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
260+
end
261+
262+
send(method_name, pragmas: { synchronous: "full" }) do |conn|
263+
assert_equal [{ "synchronous" => 2 }], conn.execute("PRAGMA synchronous")
264+
end
265+
266+
error = assert_raises(ActiveRecord::StatementInvalid) do
267+
send(method_name, pragmas: { synchronous: false }) do |conn|
268+
conn.execute("PRAGMA synchronous")
269+
end
270+
end
271+
assert_match(/unrecognized synchronous false/, error.message)
272+
end
273+
274+
def test_overriding_default_journal_size_limit_pragma
275+
method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
276+
277+
send(method_name, pragmas: { journal_size_limit: 100 }) do |conn|
278+
assert_equal [{ "journal_size_limit" => 100 }], conn.execute("PRAGMA journal_size_limit")
279+
end
280+
281+
send(method_name, pragmas: { journal_size_limit: "200" }) do |conn|
282+
assert_equal [{ "journal_size_limit" => 200 }], conn.execute("PRAGMA journal_size_limit")
283+
end
284+
285+
error = assert_raises(ActiveRecord::StatementInvalid) do
286+
send(method_name, pragmas: { journal_size_limit: false }) do |conn|
287+
conn.execute("PRAGMA journal_size_limit")
288+
end
289+
end
290+
assert_match(/undefined method `to_i'/, error.message)
291+
292+
error = assert_raises(ActiveRecord::StatementInvalid) do
293+
send(method_name, pragmas: { journal_size_limit: :false }) do |conn|
294+
conn.execute("PRAGMA journal_size_limit")
295+
end
296+
end
297+
assert_match(/undefined method `to_i'/, error.message)
298+
end
299+
300+
def test_overriding_default_mmap_size_pragma
301+
# in-memory databases never have an mmap_size
302+
if in_memory_db?
303+
with_memory_connection(pragmas: { mmap_size: 100 }) do |conn|
304+
assert_equal [], conn.execute("PRAGMA mmap_size")
305+
end
306+
307+
with_memory_connection(pragmas: { mmap_size: "200" }) do |conn|
308+
assert_equal [], conn.execute("PRAGMA mmap_size")
309+
end
310+
311+
error = assert_raises(ActiveRecord::StatementInvalid) do
312+
with_memory_connection(pragmas: { mmap_size: false }) do |conn|
313+
conn.execute("PRAGMA mmap_size")
314+
end
315+
end
316+
assert_match(/undefined method `to_i'/, error.message)
317+
318+
error = assert_raises(ActiveRecord::StatementInvalid) do
319+
with_memory_connection(pragmas: { mmap_size: :false }) do |conn|
320+
conn.execute("PRAGMA mmap_size")
321+
end
322+
end
323+
assert_match(/undefined method `to_i'/, error.message)
324+
else
325+
with_file_connection(pragmas: { mmap_size: 100 }) do |conn|
326+
assert_equal [{ "mmap_size" => 100 }], conn.execute("PRAGMA mmap_size")
327+
end
328+
329+
with_file_connection(pragmas: { mmap_size: "200" }) do |conn|
330+
assert_equal [{ "mmap_size" => 200 }], conn.execute("PRAGMA mmap_size")
331+
end
332+
333+
error = assert_raises(ActiveRecord::StatementInvalid) do
334+
with_file_connection(pragmas: { mmap_size: false }) do |conn|
335+
conn.execute("PRAGMA mmap_size")
336+
end
337+
end
338+
assert_match(/undefined method `to_i'/, error.message)
339+
340+
error = assert_raises(ActiveRecord::StatementInvalid) do
341+
with_file_connection(pragmas: { mmap_size: :false }) do |conn|
342+
conn.execute("PRAGMA mmap_size")
343+
end
344+
end
345+
assert_match(/undefined method `to_i'/, error.message)
346+
end
347+
end
348+
349+
def test_overriding_default_cache_size_pragma
350+
method_name = in_memory_db? ? :with_memory_connection : :with_file_connection
351+
352+
send(method_name, pragmas: { cache_size: 100 }) do |conn|
353+
assert_equal [{ "cache_size" => 100 }], conn.execute("PRAGMA cache_size")
354+
end
355+
356+
send(method_name, pragmas: { cache_size: "200" }) do |conn|
357+
assert_equal [{ "cache_size" => 200 }], conn.execute("PRAGMA cache_size")
358+
end
359+
360+
error = assert_raises(ActiveRecord::StatementInvalid) do
361+
send(method_name, pragmas: { cache_size: false }) do |conn|
362+
conn.execute("PRAGMA cache_size")
363+
end
364+
end
365+
assert_match(/undefined method `to_i'/, error.message)
366+
367+
error = assert_raises(ActiveRecord::StatementInvalid) do
368+
send(method_name, pragmas: { cache_size: :false }) do |conn|
369+
conn.execute("PRAGMA cache_size")
370+
end
371+
end
372+
assert_match(/undefined method `to_i'/, error.message)
373+
end
374+
375+
def test_setting_new_pragma
376+
if in_memory_db?
377+
with_memory_connection(pragmas: { temp_store: :memory }) do |conn|
378+
assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys")
379+
assert_equal [{ "journal_mode" => "memory" }], conn.execute("PRAGMA journal_mode")
380+
assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous")
381+
assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit")
382+
assert_equal [], conn.execute("PRAGMA mmap_size")
383+
assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size")
384+
assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store")
385+
end
386+
else
387+
with_file_connection(pragmas: { temp_store: :memory }) do |conn|
388+
assert_equal [{ "foreign_keys" => 1 }], conn.execute("PRAGMA foreign_keys")
389+
assert_equal [{ "journal_mode" => "wal" }], conn.execute("PRAGMA journal_mode")
390+
assert_equal [{ "synchronous" => 1 }], conn.execute("PRAGMA synchronous")
391+
assert_equal [{ "journal_size_limit" => 67108864 }], conn.execute("PRAGMA journal_size_limit")
392+
assert_equal [{ "mmap_size" => 134217728 }], conn.execute("PRAGMA mmap_size")
393+
assert_equal [{ "cache_size" => 2000 }], conn.execute("PRAGMA cache_size")
394+
assert_equal [{ "temp_store" => 2 }], conn.execute("PRAGMA temp_store")
395+
end
396+
end
397+
end
398+
399+
def test_setting_invalid_pragma
400+
if in_memory_db?
401+
warning = capture(:stderr) do
402+
with_memory_connection(pragmas: { invalid: true }) do |conn|
403+
conn.execute("PRAGMA foreign_keys")
404+
end
405+
end
406+
assert_match(/Unknown SQLite pragma: invalid/, warning)
407+
else
408+
warning = capture(:stderr) do
409+
with_file_connection(pragmas: { invalid: true }) do |conn|
410+
conn.execute("PRAGMA foreign_keys")
411+
end
412+
end
413+
assert_match(/Unknown SQLite pragma: invalid/, warning)
414+
end
415+
end
416+
178417
def test_exec_no_binds
179418
with_example_table "id int, data string" do
180419
result = @conn.exec_query("SELECT id, data FROM ex")
@@ -868,14 +1107,24 @@ def with_strict_strings_by_default
8681107

8691108
def with_file_connection(options = {})
8701109
options = options.dup
871-
db_config = ActiveRecord::Base.configurations.configurations.find { |config| !config.database.include?(":memory:") }
1110+
db_config = ActiveRecord::Base.configurations.configs_for(env_name: "arunit2", name: "primary")
8721111
options[:database] ||= db_config.database
8731112
conn = SQLite3Adapter.new(options)
8741113

8751114
yield(conn)
8761115
ensure
8771116
conn.disconnect! if conn
8781117
end
1118+
1119+
def with_memory_connection(options = {})
1120+
options = options.dup
1121+
options[:database] = ":memory:"
1122+
conn = SQLite3Adapter.new(options)
1123+
1124+
yield(conn)
1125+
ensure
1126+
conn.disconnect! if conn
1127+
end
8791128
end
8801129
end
8811130
end

0 commit comments

Comments
 (0)