Skip to content

Commit d360c55

Browse files
akostadinovakostadinov
committed
efficient piecewise LOB fetching
1 parent 5b4d2c8 commit d360c55

File tree

8 files changed

+226
-33
lines changed

8 files changed

+226
-33
lines changed

‎lib/oci8/bindtype.rb‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -245,13 +245,18 @@ def self.create(con, val, param, max_array_size)
245245
# datatype type size prec scale
246246
# -------------------------------------------------
247247
# CLOB SQLT_CLOB 4000 0 0
248-
OCI8::BindType::Mapping[:clob] = OCI8::BindType::CLOB
249-
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::NCLOB
248+
# NCLOB SQLT_CLOB 4000 0 0
249+
# Default: Fetch as String using SQLT_CHR (fast, max 2GB)
250+
# See: OCI8::lob_fetch_mode
251+
OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long
252+
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::Long
250253

251254
# datatype type size prec scale
252255
# -------------------------------------------------
253256
# BLOB SQLT_BLOB 4000 0 0
254-
OCI8::BindType::Mapping[:blob] = OCI8::BindType::BLOB
257+
# Default: Fetch as binary String using SQLT_CHR (fast, max 2GB)
258+
# See: OCI8::lob_fetch_mode
259+
OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw
255260

256261
# datatype type size prec scale
257262
# -------------------------------------------------

‎lib/oci8/oci8.rb‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ def initialize(*args)
152152
end
153153

154154
@prefetch_rows = 100
155+
@lob_prefetch_size = 0 # 0 means use Oracle default (disabled)
155156
@username = nil
156157
end
157158

@@ -333,6 +334,27 @@ def prefetch_rows=(num)
333334
@prefetch_rows = num
334335
end
335336

337+
# Sets the LOB prefetch size in bytes. Only used when lob_fetch_mode is :locator.
338+
# i.e. sets OCI_ATTR_DEFAULT_LOBPREFETCH_SIZE on the session handle.
339+
# The default value is 0 (disabled).
340+
#
341+
# When set to a non-zero value (e.g., 65536 for 64KB), Oracle prefetches
342+
# LOB data along with the row if the LOB size is <= this value.
343+
# This reduces network round trips when fetching small to medium LOBs.
344+
#
345+
# This is a session-wide setting that applies to all LOB columns fetched
346+
# from this connection.
347+
#
348+
# It has no effect when lob_fetch_mode is :long_as_string (the default).
349+
#
350+
# @param [Integer] size prefetch size in bytes (0 to disable)
351+
# @see OCI8::lob_fetch_mode=
352+
# @note Requires Oracle 11g or later
353+
def lob_prefetch_size=(size)
354+
@lob_prefetch_size = size
355+
@session_handle.send(:attr_set_ub4, 438, size)
356+
end
357+
336358
# @private
337359
def inspect
338360
"#<OCI8:#{username}>"
@@ -370,6 +392,48 @@ def self.client_charset_name
370392
@@client_charset_name
371393
end
372394

395+
# Returns the current LOB fetch mode.
396+
#
397+
# @return [Symbol] :long_as_string or :locator
398+
# @see lob_fetch_mode=
399+
def self.lob_fetch_mode
400+
@@lob_fetch_mode ||= :long_as_string
401+
end
402+
403+
# Sets the LOB fetch mode.
404+
#
405+
# - +:long_as_string+ (default): Fetch LOBs as Strings using
406+
# Runtime Data Allocation and Piecewise Operations in OCI.
407+
# Fastest and most efficient fetch but limited to 2GB LOBs.
408+
#
409+
# - +:locator+: Fetch LOB locators (OCI8::CLOB/BLOB objects).
410+
# Calls OCILobRead2() on the lob fields individually.
411+
# Required for LOBs > 2GB, random access, or read/write operations.
412+
# @conn.lob_prefetch_size may reduce network roundtrips but my
413+
# unscientific testing only showed performance degradation.
414+
#
415+
# @param [Symbol] mode :long_as_string or :locator
416+
# @return [Symbol] the mode that was set
417+
# @see https://github.com/oracle/odpi/issues/163
418+
def self.lob_fetch_mode=(mode)
419+
unless [:long_as_string, :locator].include?(mode)
420+
raise ArgumentError, "lob_fetch_mode must be :long_as_string or :locator"
421+
end
422+
423+
case mode
424+
when :long_as_string
425+
OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long
426+
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::Long
427+
OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw
428+
when :locator
429+
OCI8::BindType::Mapping[:clob] = OCI8::BindType::CLOB
430+
OCI8::BindType::Mapping[:nclob] = OCI8::BindType::NCLOB
431+
OCI8::BindType::Mapping[:blob] = OCI8::BindType::BLOB
432+
end
433+
434+
@@lob_fetch_mode = mode
435+
end
436+
373437
if OCI8.oracle_client_version >= OCI8::ORAVER_11_1
374438
# Returns send timeout in seconds.
375439
# Zero means no timeout.

‎test/test_clob.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,17 @@ class TestCLob < Minitest::Test
66

77
def setup
88
@conn = get_oci8_connection
9+
# This test needs LOB locators for read/write operations
10+
OCI8.lob_fetch_mode = :locator
911
drop_table('test_table')
1012
@conn.exec('CREATE TABLE test_table (filename VARCHAR2(40), content CLOB)')
1113
end
1214

15+
def teardown
16+
# Restore default mode
17+
OCI8.lob_fetch_mode = :long_as_string if @conn
18+
end
19+
1320
def test_insert
1421
filename = File.basename($lobfile)
1522
@conn.exec("DELETE FROM test_table WHERE filename = :1", filename)

‎test/test_dbi_clob.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,17 @@ class TestDbiCLob < Minitest::Test
66

77
def setup
88
@dbh = get_dbi_connection()
9+
# This test needs LOB locators for read/write operations
10+
OCI8.lob_fetch_mode = :locator
911
drop_table('test_table')
1012
@dbh.execute('CREATE TABLE test_table (filename VARCHAR2(40), content CLOB)')
1113
end
1214

15+
def teardown
16+
# Restore default mode
17+
OCI8.lob_fetch_mode = :long_as_string if @dbh
18+
end
19+
1320
def test_insert
1421
filename = File.basename($lobfile)
1522

‎test/test_encoding.rb‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@
44
class TestEncoding < Minitest::Test
55
def setup
66
@conn = get_oci8_connection
7+
# This test needs LOB locators for read/write operations
8+
OCI8.lob_fetch_mode = :locator
9+
end
10+
11+
def teardown
12+
# Restore default mode
13+
OCI8.lob_fetch_mode = :long_as_string if @conn
714
end
815

916
def test_select

‎test/test_large_lob.rb‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/usr/bin/env ruby
2+
#
3+
# Test to verify that 512KB LOBs can be fetched using both :long_as_string and :locator modes
4+
#
5+
6+
require 'oci8'
7+
require_relative 'config'
8+
9+
class TestLargeLob < Minitest::Test
10+
def setup
11+
@conn = get_oci8_connection
12+
@saved_lob_fetch_mode = OCI8.lob_fetch_mode
13+
drop_table('test_large_lob')
14+
@conn.exec(<<-SQL)
15+
CREATE TABLE test_large_lob (
16+
id NUMBER,
17+
clob_data CLOB,
18+
blob_data BLOB
19+
)
20+
SQL
21+
end
22+
23+
def teardown
24+
return unless @conn
25+
26+
OCI8.lob_fetch_mode = @saved_lob_fetch_mode
27+
drop_table('test_large_lob')
28+
@conn.logoff
29+
end
30+
31+
def test_512kb_lob_roundtrip_with_long_interface
32+
size_kb = 512
33+
clob_data = generate_text_data(size_kb)
34+
blob_data = generate_binary_data(size_kb)
35+
insert_lob_row(1, clob_data, blob_data)
36+
37+
# Fetch using :long_as_string mode (LONG interface)
38+
OCI8.lob_fetch_mode = :long_as_string
39+
cursor = @conn.exec("SELECT id, clob_data, blob_data FROM test_large_lob WHERE id = 1")
40+
row = cursor.fetch
41+
cursor.close
42+
43+
# Verify we got String objects with correct data
44+
assert_equal 1, row[0]
45+
assert_instance_of String, row[1], "CLOB should be fetched as String"
46+
assert_instance_of String, row[2], "BLOB should be fetched as String"
47+
assert_equal size_kb * 1024, row[1].size, "CLOB size should match"
48+
assert_equal size_kb * 1024, row[2].size, "BLOB size should match"
49+
assert_equal clob_data, row[1], "CLOB data should match (verifies chunk order)"
50+
assert_equal blob_data, row[2], "BLOB data should match (verifies chunk order)"
51+
end
52+
53+
def test_512kb_lob_roundtrip_with_locator_mode
54+
size_kb = 512
55+
clob_data = generate_text_data(size_kb)
56+
blob_data = generate_binary_data(size_kb)
57+
insert_lob_row(2, clob_data, blob_data)
58+
59+
# Fetch using :locator mode (LOB locators)
60+
OCI8.lob_fetch_mode = :locator
61+
cursor = @conn.exec("SELECT id, clob_data, blob_data FROM test_large_lob WHERE id = 2")
62+
row = cursor.fetch
63+
cursor.close
64+
65+
# Verify we got LOB locator objects
66+
assert_equal 2, row[0]
67+
assert_instance_of OCI8::CLOB, row[1], "CLOB should be fetched as locator"
68+
assert_instance_of OCI8::BLOB, row[2], "BLOB should be fetched as locator"
69+
70+
# Read from locators and verify data
71+
fetched_clob_data = row[1].read
72+
fetched_blob_data = row[2].read
73+
assert_equal size_kb * 1024, fetched_clob_data.size, "CLOB size should match"
74+
assert_equal size_kb * 1024, fetched_blob_data.size, "BLOB size should match"
75+
assert_equal clob_data, fetched_clob_data, "CLOB data should match (verifies chunk order)"
76+
assert_equal blob_data, fetched_blob_data, "BLOB data should match (verifies chunk order)"
77+
end
78+
79+
private
80+
81+
def generate_text_data(size_kb, seed = 42)
82+
# hex encoding: 1 byte -> 2 hex characters
83+
binary_size_kb = (size_kb / 2.0).ceil
84+
binary_data = generate_binary_data(binary_size_kb, seed)
85+
hex_data = binary_data.unpack1('H*')
86+
hex_data[0, size_bytes] # truncate to exact requested size
87+
end
88+
89+
def generate_binary_data(size_kb, seed = 42)
90+
Random.new(seed).bytes(size_kb * 1024)
91+
end
92+
93+
def insert_lob_row(id, clob_data, blob_data)
94+
cursor = @conn.parse("INSERT INTO test_large_lob VALUES (:1, :2, :3)")
95+
cursor.exec(id, OCI8::CLOB.new(@conn, clob_data), OCI8::BLOB.new(@conn, blob_data))
96+
cursor.close
97+
@conn.commit
98+
end
99+
end

‎test/test_object.rb‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,14 @@ class TestObj1 < Minitest::Test
5959

6060
def setup
6161
@conn = get_oci8_connection
62+
# This test needs LOB locators for read operations on object attributes
63+
OCI8.lob_fetch_mode = :locator
6264
RbTestObj.default_connection = @conn
6365
end
6466

6567
def teardown
68+
# Restore default mode
69+
OCI8.lob_fetch_mode = :long_as_string
6670
@conn.logoff
6771
end
6872

‎test/test_oci8.rb‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ def test_rename
4141
]
4242

4343
def test_long_type
44-
clob_bind_type = OCI8::BindType::Mapping[:clob]
45-
blob_bind_type = OCI8::BindType::Mapping[:blob]
4644
initial_cunk_size = OCI8::BindType::Base.initial_chunk_size
4745
begin
4846
OCI8::BindType::Base.initial_chunk_size = 5
4947
@conn.prefetch_rows = LONG_TEST_DATA.size / 3
5048
drop_table('test_table')
5149
ascii_enc = Encoding.find('US-ASCII')
5250
0.upto(1) do |i|
51+
# First part of test uses LOB locators
52+
OCI8.lob_fetch_mode = :locator
5353
if i == 0
5454
@conn.exec("CREATE TABLE test_table (id number(38), long_column long, clob_column clob)")
5555
cursor = @conn.parse('insert into test_table values (:1, :2, :3)')
@@ -108,39 +108,35 @@ def test_long_type
108108
assert_nil(cursor.fetch)
109109
cursor.close
110110

111-
begin
112-
OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long
113-
OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw
114-
cursor = @conn.parse('SELECT * from test_table order by id')
115-
cursor.exec
116-
LONG_TEST_DATA.each_with_index do |data, index|
117-
row = cursor.fetch
118-
assert_equal(index, row[0])
119-
if data.nil?
120-
assert_nil(row[1])
121-
assert_nil(row[2])
122-
elsif data.empty?
123-
# '' is inserted to the long or long raw column as null.
124-
assert_nil(row[1])
125-
# '' is inserted to the clob or blob column as an empty clob.
126-
# However it is fetched as nil.
127-
assert_nil(row[2])
128-
else
129-
assert_equal(data, row[1])
130-
assert_equal(data, row[2])
131-
assert_equal(enc, row[1].encoding)
132-
assert_equal(enc, row[2].encoding)
133-
end
111+
# Second part of test uses Long bind type (fetch as strings)
112+
OCI8.lob_fetch_mode = :long_as_string
113+
cursor = @conn.parse('SELECT * from test_table order by id')
114+
cursor.exec
115+
LONG_TEST_DATA.each_with_index do |data, index|
116+
row = cursor.fetch
117+
assert_equal(index, row[0])
118+
if data.nil?
119+
assert_nil(row[1])
120+
assert_nil(row[2])
121+
elsif data.empty?
122+
# '' is inserted to the long or long raw column as null.
123+
assert_nil(row[1])
124+
# '' is inserted to the clob or blob column as an empty clob.
125+
# However it is fetched as nil.
126+
assert_nil(row[2])
127+
else
128+
assert_equal(data, row[1])
129+
assert_equal(data, row[2])
130+
assert_equal(enc, row[1].encoding)
131+
assert_equal(enc, row[2].encoding)
134132
end
135-
assert_nil(cursor.fetch)
136-
cursor.close
137-
ensure
138-
OCI8::BindType::Mapping[:clob] = clob_bind_type
139-
OCI8::BindType::Mapping[:blob] = blob_bind_type
140133
end
134+
assert_nil(cursor.fetch)
135+
cursor.close
141136
drop_table('test_table')
142137
end
143138
ensure
139+
OCI8.lob_fetch_mode = :long_as_string
144140
OCI8::BindType::Base.initial_chunk_size = initial_cunk_size
145141
end
146142
drop_table('test_table')
@@ -396,6 +392,8 @@ def test_binary_float
396392
def test_clob_nclob_and_blob
397393
return if OCI8::oracle_client_version < OCI8::ORAVER_8_1
398394

395+
# This test needs LOB locators
396+
OCI8.lob_fetch_mode = :locator
399397
drop_table('test_table')
400398
sql = <<-EOS
401399
CREATE TABLE test_table (id number(5), C CLOB, NC NCLOB, B BLOB)
@@ -435,6 +433,8 @@ def test_clob_nclob_and_blob
435433
assert_nil(cursor.fetch)
436434
cursor.close
437435
drop_table('test_table')
436+
ensure
437+
OCI8.lob_fetch_mode = :long_as_string
438438
end
439439

440440
def test_select_number

0 commit comments

Comments
 (0)