Merge pull request #1173 from oracle-devrel/uschwinn-patch-23

Uschwinn patch 23

Author: JesusGitHubOracle

data-platform/core-converged-db/ai-vector-search/files/0_setup_ts_user_directory.sql

@@ -0,0 +1,22 @@
+REM Creation of tablespace, user and directory
+
+-- connect as user sys to FREEPDB1
+
+-- create tablespace
+
+create bigfile tablespace TBS_VECTOR datafile size 256M autoextend on maxsize 2G;
+
+-- create user with the new role DB_DEVELOPER_ROLE
+DROP USER vector_user cascade;
+
+create user vector_user identified by "Oracle_4U"
+default tablespace TBS_VECTOR temporary tablespace TEMP
+quota unlimited on TBS_VECTOR;
+
+grant create mining model to vector_user;
+grant DB_DEVELOPER_ROLE to vector_user;
+
+-- create directory
+
+CREATE OR REPLACE DIRECTORY dm_dump as '&directorypath';
+GRANT all ON DIRECTORY dm_dump TO vector_user;

data-platform/core-converged-db/ai-vector-search/files/1_externaltable.sql

@@ -0,0 +1,27 @@
+REM Create an external table and check the data set
+ 
+-- copy the data set in the directory DM_DUMP
+-- dataset_200K.txt is located in: https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/fro8fl9kuqli/b/AIVECTORS/o/dataset_200K.txt
+
+-- connect as user VECTOR_USER and create an external table
+
+connect vector_user/Oracle_4U@FREEPDB1
+
+CREATE TABLE if not exists CCNEWS_TMP (sentence VARCHAR2(4000))
+  ORGANIZATION EXTERNAL (TYPE ORACLE_LOADER DEFAULT DIRECTORY dm_dump
+                         ACCESS PARAMETERS
+                           (RECORDS DELIMITED BY 0x'0A'
+                            READSIZE 100000000
+                            FIELDS (sentence CHAR(4000)))
+                         LOCATION (sys.dm_dump:'dataset_200K.txt'))
+  PARALLEL
+  REJECT LIMIT UNLIMITED;
+
+-- Check that the external table is correct
+
+select count(*) from CCNEWS_TMP; 
+
+-- Check the 4 first rows
+
+select * from CCNEWS_TMP where rownum < 4;
+

data-platform/core-converged-db/ai-vector-search/files/2_load_vector_embedding_in_db.sql

@@ -0,0 +1,16 @@
+REM  load a vector embedding model into the database: we can load a ONNX model inside the database using DBMS_VECTOR package 
+
+/* 
+The ONNX file in this example was generated by Oracle OML4Py utility from the all-MiniLM-L6-v2 sentence transformer model. 
+It can be downloaded from https://objectstorage.eu-frankfurt-1.oraclecloud.com/n/fro8fl9kuqli/b/AIVECTORS/o/all-MiniLM-L6-v2.onnx
+Copy the all-MiniLM-L6-v2.onnx file to the path corresponding to the DM_DUMP directory.
+  */ 
+
+-- connect to the VECTOR_USER 
+
+connect vector_user/Oracle_4U@FREEPDB1
+
+
+-- load the ONNX model in the database
+
+EXECUTE DBMS_VECTOR.LOAD_ONNX_MODEL('DM_DUMP','all-MiniLM-L6-v2.onnx','doc_model');

data-platform/core-converged-db/ai-vector-search/files/3_check_model.sql

@@ -0,0 +1,23 @@
+REM  Load the ONNX model into the database with DBMS_VECTOR
+
+connect vector_user/Oracle_4U@FREEPDB1
+
+col model_name format a12
+col mining_function format a12
+col algorithm format a12
+col attribute_name format a20
+col data_type format a30
+col vector_info format a30
+set lines 120
+
+SELECT model_name, mining_function, algorithm,
+algorithm_type, model_size
+FROM user_mining_models
+WHERE model_name = 'DOC_MODEL'
+ORDER BY model_name;
+
+
+SELECT model_name, attribute_name, attribute_type, data_type, vector_info
+FROM user_mining_model_attributes
+WHERE model_name = 'DOC_MODEL'
+ORDER BY attribute_name;

data-platform/core-converged-db/ai-vector-search/files/4_calculate_vector_embeddings.sql

@@ -0,0 +1,25 @@
+REM  Calculate embeddings directly in the database
+
+-- connect to the VECTOR_USER 
+
+connect vector_user/Oracle_4U@FREEPDB1
+
+-- create table CCNEWS
+create table if not exists CCNEWS (
+    id number(10) not null,
+    info VARCHAR2(4000),
+    vec VECTOR
+);
+
+
+-- Use the doc_model previously loaded to calculate the vector embeddings
+-- Please plan additional time to complete this statement 
+
+insert into CCNEWS (id, info, vec)
+select rownum,
+       sentence,
+       TO_VECTOR(VECTOR_EMBEDDING(doc_model USING sentence as data))
+from CCNEWS_TMP;
+
+
+commit;

data-platform/core-converged-db/ai-vector-search/files/5_semantic_search_query.sql

@@ -0,0 +1,27 @@
+REM  Use semantic search query using vectors
+
+
+-- connect to the VECTOR_USER 
+
+connect vector_user/Oracle_4U@FREEPDB1
+
+-- use "COSINE" parameter in VECTOR_DISTANCE to calculate the distance between two vectors
+
+set timing on
+col info format a90
+set lines 120
+
+select id, info
+from CCNEWS
+order by vector_distance(vec, TO_VECTOR(VECTOR_EMBEDDING(doc_model USING 'little red corvette' as data)), COSINE)
+fetch approx first 5 rows only;
+
+
+-- Check the execution plan
+
+set autotrace traceonly explain
+
+select id, info
+from CCNEWS
+order by vector_distance(vec, TO_VECTOR(VECTOR_EMBEDDING(doc_model USING 'little red corvette' as data)), COSINE)
+fetch approx first 5 rows only;

data-platform/core-converged-db/ai-vector-search/files/README.md

@@ -10,7 +10,7 @@ Use the scripts to learn basics about AI Vector Search.
 
 # How to use this asset?
 
-See the README in the files folder.
+Please use the posting [Getting started with vectors in 23ai](https://blogs.oracle.com/coretec/post/getting-started-with-vectors-in-23ai) for information about the scripts.
 
 # Useful Links