Skip to content

Commit 1b53b5e

Browse files
tom-dyartom-dyar
committed
sync: update from internal GitLab repository
Content updated: Directories: - objectscript/ Synced at: 2025-08-02 23:06:29
1 parent a36b7da commit 1b53b5e

File tree

2 files changed

+117
-83
lines changed

2 files changed

+117
-83
lines changed

‎objectscript/RAG/IFindSetup.CLS

Lines changed: 24 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2,47 +2,29 @@
22
Class RAG.IFindSetup
33
{
44

5-
/// Create a new table with iFind support since we can't alter existing table
5+
/// Create persistent class table for iFind functionality
66
ClassMethod CreateIFindTable() As %Status
77
{
88
Try {
9-
Write "Creating new table with iFind support...",!
9+
Write "Setting up iFind table using persistent class...",!
1010

11-
// Create a new table that mirrors SourceDocuments but with iFind
12-
&sql(CREATE TABLE RAG.SourceDocumentsIFind (
13-
doc_id VARCHAR(200) PRIMARY KEY,
14-
title VARCHAR(500),
15-
text_content LONGVARCHAR,
16-
authors LONGVARCHAR,
17-
keywords LONGVARCHAR,
18-
embedding VARCHAR(32000),
19-
created_at TIMESTAMP
20-
))
21-
22-
If SQLCODE '= 0 {
23-
Write "Error creating table: SQLCODE=", SQLCODE, " MSG=", %msg,!
24-
Return $$$ERROR($$$GeneralError, "Failed to create table")
11+
// Compile the SourceDocumentsWithIFind class to create table
12+
Set status = $System.OBJ.Compile("RAG.SourceDocumentsWithIFind.cls", "ck")
13+
If $$$ISERR(status) {
14+
Write "Error compiling class: ", $System.Status.GetErrorText(status),!
15+
Return status
2516
}
2617

27-
Write "Table created successfully",!
28-
29-
// Now add the iFind index using ALTER TABLE
30-
Write "Adding iFind index...",!
31-
&sql(ALTER TABLE RAG.SourceDocumentsIFind ADD FULLTEXT INDEX idx_ifind (text_content))
18+
Write "✅ iFind table class compiled successfully",!
3219

33-
If SQLCODE '= 0 {
34-
Write "Error creating iFind index: SQLCODE=", SQLCODE, " MSG=", %msg,!
35-
// Try alternative syntax
36-
Write "Trying alternative syntax...",!
37-
&sql(CREATE FULLTEXT INDEX idx_ifind ON RAG.SourceDocumentsIFind (text_content))
38-
39-
If SQLCODE '= 0 {
40-
Write "Still failed: SQLCODE=", SQLCODE, " MSG=", %msg,!
41-
Return $$$ERROR($$$GeneralError, "Failed to create iFind index")
42-
}
20+
// Test that the table was created
21+
&sql(SELECT COUNT(*) INTO :count FROM RAG.SourceDocumentsWithIFind)
22+
If SQLCODE = 0 {
23+
Write "✅ Table created and accessible",!
24+
} Else {
25+
Write "Table creation verification failed: SQLCODE=", SQLCODE,!
4326
}
4427

45-
Write "✅ iFind index created successfully!",!
4628
Return $$$OK
4729

4830
} Catch ex {
@@ -57,8 +39,11 @@ ClassMethod CopyDataToIFindTable() As %Status
5739
Try {
5840
Write "Copying data to iFind table...",!
5941

60-
&sql(INSERT INTO RAG.SourceDocumentsIFind
61-
SELECT * FROM RAG.SourceDocuments)
42+
// Use INSERT with explicit column mapping for compatibility
43+
&sql(INSERT INTO RAG.SourceDocumentsWithIFind
44+
(doc_id, title, text_content, authors, keywords, embedding, created_at)
45+
SELECT doc_id, title, text_content, authors, keywords, embedding, created_at
46+
FROM RAG.SourceDocuments)
6247

6348
If SQLCODE = 0 {
6449
Write "✅ Copied ", %ROWCOUNT, " documents",!
@@ -74,16 +59,16 @@ ClassMethod CopyDataToIFindTable() As %Status
7459
}
7560
}
7661

77-
/// Test iFind search using %CONTAINS
62+
/// Test iFind search using %FIND
7863
ClassMethod TestIFindSearch(searchText As %String) As %Status
7964
{
8065
Try {
8166
Write !,"Searching for: ", searchText,!,!
8267

8368
&sql(DECLARE C1 CURSOR FOR
8469
SELECT TOP 10 doc_id, title
85-
FROM RAG.SourceDocumentsIFind
86-
WHERE %CONTAINS(text_content, :searchText))
70+
FROM RAG.SourceDocumentsWithIFind
71+
WHERE %FIND(text_content, :searchText) > 0)
8772

8873
&sql(OPEN C1)
8974

@@ -131,8 +116,8 @@ ClassMethod Setup() As %Status
131116

132117
Write !,"✅ Setup complete!",!
133118
Write "Update hybrid_ifind_rag/pipeline.py to use:",!
134-
Write " FROM RAG.SourceDocumentsIFind",!
135-
Write " WHERE %CONTAINS(text_content, ?)",!
119+
Write " FROM RAG.SourceDocumentsWithIFind",!
120+
Write " WHERE %FIND(text_content, ?) > 0",!
136121

137122
Return $$$OK
138123
}

‎objectscript/RAG/SourceDocumentsWithIFind.CLS

Lines changed: 93 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,96 @@
1-
Class RAG.SourceDocumentsWithIFind Extends %Persistent [ SqlTableName = SourceDocumentsIFind ]
1+
/// RAG SourceDocuments table with iFind support
2+
/// This class provides a table structure compatible with iFind functionality
3+
Class RAG.SourceDocumentsWithIFind Extends %Persistent
24
{
3-
Property doc_id As %String(MAXLEN = 255) [ Required ];
4-
Property title As %String(MAXLEN = 1000);
5-
Property text_content As %Stream.GlobalCharacter;
6-
Property embedding As %String(MAXLEN = 60000);
7-
Property created_at As %TimeStamp;
8-
9-
Index DocIdIndex On doc_id [ Unique ];
10-
11-
// This is the key - %iFind.Index.Basic on the STREAM field
12-
Index TextContentFTI On (text_content) As %iFind.Index.Basic(
13-
LANGUAGE = "en",
14-
LOWER = 1,
15-
INDEXOPTION = 2 // Enable stemming and decompounding
16-
);
17-
18-
Storage Default
19-
{
20-
<Data name="SourceDocumentsIFindDefaultData">
21-
<Value name="1">
22-
<Value>%%CLASSNAME</Value>
23-
</Value>
24-
<Value name="2">
25-
<Value>doc_id</Value>
26-
</Value>
27-
<Value name="3">
28-
<Value>title</Value>
29-
</Value>
30-
<Value name="4">
31-
<Value>text_content</Value>
32-
</Value>
33-
<Value name="5">
34-
<Value>embedding</Value>
35-
</Value>
36-
<Value name="6">
37-
<Value>created_at</Value>
38-
</Value>
39-
</Data>
40-
<DataLocation>^RAG.SourceDocumentsIFindD</DataLocation>
41-
<DefaultData>SourceDocumentsIFindDefaultData</DefaultData>
42-
<IdLocation>^RAG.SourceDocumentsIFindD</IdLocation>
43-
<IndexLocation>^RAG.SourceDocumentsIFindI</IndexLocation>
44-
<StreamLocation>^RAG.SourceDocumentsIFindS</StreamLocation>
45-
<Type>%Storage.Persistent</Type>
5+
6+
/// Document identifier
7+
Property doc_id As %String(MAXLEN = 255) [ Required ];
8+
9+
/// Document title
10+
Property title As %String(MAXLEN = 500);
11+
12+
/// Full text content - iFind works on any text field automatically
13+
Property text_content As %Stream.GlobalCharacter;
14+
15+
/// Document authors
16+
Property authors As %String(MAXLEN = 1000);
17+
18+
/// Document keywords
19+
Property keywords As %String(MAXLEN = 1000);
20+
21+
/// Vector embedding as string
22+
Property embedding As %String(MAXLEN = 60000);
23+
24+
/// Creation timestamp
25+
Property created_at As %TimeStamp [ InitialExpression = {$ZDateTime($Horolog,3)} ];
26+
27+
/// Index on document ID for fast lookups
28+
Index DocIdIndex On doc_id [ Unique ];
29+
30+
/// Index on creation timestamp for temporal queries
31+
Index CreatedAtIndex On created_at;
32+
33+
/// Test iFind search functionality
34+
ClassMethod TestSearch(searchText As %String = "diabetes") As %Status
35+
{
36+
Set tSC = $$$OK
37+
Try {
38+
Write "Testing iFind search for: ", searchText, !
39+
40+
// Use %FIND function which works with iFind
41+
&sql(SELECT COUNT(*) INTO :count
42+
FROM RAG.SourceDocumentsWithIFind
43+
WHERE %FIND(text_content, :searchText) > 0)
44+
45+
If SQLCODE = 0 {
46+
Write "✓ Found ", count, " documents matching '", searchText, "'", !
47+
} Else {
48+
Write "Search completed with SQLCODE: ", SQLCODE, !
49+
}
50+
51+
} Catch ex {
52+
Set tSC = ex.AsStatus()
53+
Write "Error during search: ", ex.DisplayString(), !
4654
}
55+
56+
Quit tSC
57+
}
58+
59+
/// Storage definition for the class
60+
Storage Default
61+
{
62+
<Data name="SourceDocumentsWithIFindDefaultData">
63+
<Value name="1">
64+
<Value>%%CLASSNAME</Value>
65+
</Value>
66+
<Value name="2">
67+
<Value>doc_id</Value>
68+
</Value>
69+
<Value name="3">
70+
<Value>title</Value>
71+
</Value>
72+
<Value name="4">
73+
<Value>text_content</Value>
74+
</Value>
75+
<Value name="5">
76+
<Value>authors</Value>
77+
</Value>
78+
<Value name="6">
79+
<Value>keywords</Value>
80+
</Value>
81+
<Value name="7">
82+
<Value>embedding</Value>
83+
</Value>
84+
<Value name="8">
85+
<Value>created_at</Value>
86+
</Value>
87+
</Data>
88+
<DataLocation>^RAG.SourceDocumentsWithIFindD</DataLocation>
89+
<DefaultData>SourceDocumentsWithIFindDefaultData</DefaultData>
90+
<IdLocation>^RAG.SourceDocumentsWithIFindD</IdLocation>
91+
<IndexLocation>^RAG.SourceDocumentsWithIFindI</IndexLocation>
92+
<StreamLocation>^RAG.SourceDocumentsWithIFindS</StreamLocation>
93+
<Type>%Storage.Persistent</Type>
94+
}
95+
4796
}

0 commit comments

Comments
 (0)