Merge pull request #4 from martygubar/main

centralized credential creation and added RAG example

Author: malcherm

azure-cli/config

@@ -10,6 +10,7 @@ ADB_NAME="quickstart"
 # database
 VNET_NAME="dev-vnet"
 VNET_PREFIX="19x.xxx.0.0/16"
+
 # subnet for the database
 SUBNET_NAME="dev-sn-db"
 SUBNET_PREFIX="19x.xxx.1.0/24"
@@ -34,5 +35,5 @@ STORAGE_CONTAINER_NAME="adb-sample"
 ## IDENTITIES
 # This identity will be used for your VM. The password will also be used for the database ADMIN user
 USER_NAME="adb"
-# The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character.
+--The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character
 USER_PASSWORD=""

azure-cli/config.default

@@ -9,16 +9,14 @@ ADB_NAME="quickstart"
 ## NETWORKING
 # database
 VNET_NAME="dev-vnet"
-#VNET_PREFIX="192.168.0.0/16"
 VNET_PREFIX="19x.xxx.0.0/16"
+
 # subnet for the database
 SUBNET_NAME="dev-sn-db"
-#SUBNET_PREFIX="192.168.1.0/24"
 SUBNET_PREFIX="19x.xxx.1.0/24"
 
 # client subnet
 SUBNET2_NAME="dev-sn-client"
-#SUBNET2_PREFIX="192.168.2.0/24"
 SUBNET2_PREFIX="19x.xxx.2.0/24"
 
 #network security group
@@ -37,4 +35,5 @@ STORAGE_CONTAINER_NAME="adb-sample"
 ## IDENTITIES
 # This identity will be used for your VM. The password will also be used for the database ADMIN user
 USER_NAME="adb"
-USER_PASSWORD="your-complex-password"
+--The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character
+USER_PASSWORD=""

sql/README.md

@@ -16,6 +16,7 @@ Try out these scripts to learn how to get started using Autonomous Database. Sim
 
 |Script|Description|
 |----|---|
+|[credential-create.sql](credential-create.sql)|Autonomous Database credentials contain the secret keys used to connect to services - like Azure OpenAI. This script creates those credentials. It's called by multiple scripts listed below |
 |[data-create-sample-schema.sql](data-create-sample-schema.sql)|Create a sample user and install sample data|
 |[data-create-synthetic-data.sql](data-create-synthetic-data.sql)|Use AI to generate sample data sets|
 |[data-import-from-datalake.sql](data-create-synthetic-data.sql)|Import sample data from Azure Data Lake. Sample data was uploaded using the [`create-all-resources.sh`](../azure-cli/create-all-resources.sh)and [`create-data-lake-storage.sh`](../azure-cli/create-data-lake-storage.sh) scripts. You can run [`show-data-lake-storage-info.sh`](../azure-cli/show-data-lake-storage-info.sh) to get connection information to the storage container.|
@@ -43,14 +44,16 @@ chmod 600 config.sql
 |**Select AI and GenAI**|
 |AZURE_OPENAI_RESOURCE_NAME|Name of the Azure OpenAI endpoint|'dev-adb-azure-openai'|
 |AZURE_OPENAI_ENDPOINT|Your Azure OpenAI endpoint (server name only)|'my-openai.openai.azure.com'|
-|AZURE_OPENAI_DEPLOYMENT_NAME|Your Azure OpenAI deployment name|'gpt-4o'|
+|AZURE_OPENAI_DEPLOYMENT_NAME|Your Azure OpenAI deployment name. This is used for NL2SQL and AI SQLfunctions|'gpt-4o'|
+|AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME|The Azure OpenAI deployment that uses an embedding model. This is used for creating vector embeddings.|'text-embedding-ada-002'|
 |AZURE_OPENAI_KEY|Azure OpenAI secret key|'3Cu9AB...H53'|
 |AZURE_OPENAI_PROFILE_NAME|The Select AI profile that will reference your Azure OpenAI deployment|'gpt4o'|
 |AZURE_OPENAI_CREDENTIAL_NAME|The database credential that will be used to connect to Azure OpenAI|'azure_cred4o'|
 |**Azure Data Lake Storage**|
 |STORAGE_ACCOUNT_NAME|Name of your Azure Data Lake Storage Gen 2 account. You can run [`show-data-lake-storage-info.sh`](../azure-cli/show-data-lake-storage-info.sh) to get storage details|'mysamplestorage'|
 |STORAGE_URL|Azure data lake storage URL|'https://mysamplestorage.blob.core.windows.net/adb-sample'
 |STORAGE_KEY|The secret key used to connecto Azure Data Lake Storage|'dJVNxq1YTT...jp/g=='
+|STORAGE_CREDENTIAL_NAME|The name of the Autonomous Database credential that's used to connect to Azure Data Lake Storage Gen 2|'adls_cred'|
 
 You can find the Azure OpenAI settings in the Azure OpenAI Studio:
 ![Azure OpenAI settings](images/azure-openai.png)

sql/config-default.sql

@@ -9,7 +9,8 @@
 define CONN='your-database-connection-string'  
 -- the database user that will own the sample schema
 define USER_NAME='moviestream' 
-define USER_PASSWORD='your-strong-password' 
+--The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character
+define USER_PASSWORD='' 
 --
 -- GENAI
 --
@@ -19,6 +20,8 @@ define AZURE_OPENAI_ENDPOINT='your-azure-openai-endpoint'
 define AZURE_OPENAI_RESOURCE_NAME='your-azure-openai-resourcename'
 -- Azure OpenAI deployment Name
 define AZURE_OPENAI_DEPLOYMENT_NAME='your-azure-openai-deployment-name'
+-- Azure OpenAI Embedding deployment name. This is used for creating embeddings for RAG
+define AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME='your-azure-openai-embedding-deployment-name'
 -- Azure OpenAI key
 define AZURE_OPENAI_KEY='your-azure-openai-key'
 
@@ -32,4 +35,6 @@ define AZURE_OPENAI_CREDENTIAL_NAME='azure_cred4o'
 -- Get this information by running ../azure-cli/show-data-lake-storage-info.sh
 define STORAGE_KEY='your-azure-data-lake-storage-key' 
 define STORAGE_ACCOUNT_NAME='your-azure-data-lake-storage-account-name'
-define STORAGE_URL='https://your-storage-url/adb-sample'
+define STORAGE_URL='https://your-storage-url'
+-- You can leave this default
+define STORAGE_CREDENTIAL_NAME='adls_cred'

sql/config.sql

@@ -9,8 +9,7 @@
 define CONN='your-database-connection-string'  
 -- the database user that will own the sample schema
 define USER_NAME='moviestream' 
--- Password for the database user
--- The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character.
+--The password must be between 12 and 30 characters long and must include at least one uppercase letter, one lowercase letter, and one numeric character
 define USER_PASSWORD='' 
 --
 -- GENAI
@@ -21,6 +20,8 @@ define AZURE_OPENAI_ENDPOINT='your-azure-openai-endpoint'
 define AZURE_OPENAI_RESOURCE_NAME='your-azure-openai-resourcename'
 -- Azure OpenAI deployment Name
 define AZURE_OPENAI_DEPLOYMENT_NAME='your-azure-openai-deployment-name'
+-- Azure OpenAI Embedding deployment name. This is used for creating embeddings for RAG
+define AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME='your-azure-openai-embedding-deployment-name'
 -- Azure OpenAI key
 define AZURE_OPENAI_KEY='your-azure-openai-key'
 
@@ -34,4 +35,6 @@ define AZURE_OPENAI_CREDENTIAL_NAME='azure_cred4o'
 -- Get this information by running ../azure-cli/show-data-lake-storage-info.sh
 define STORAGE_KEY='your-azure-data-lake-storage-key' 
 define STORAGE_ACCOUNT_NAME='your-azure-data-lake-storage-account-name'
-define STORAGE_URL='https://your-storage-url/adb-sample'
+define STORAGE_URL='https://your-storage-url'
+-- You can leave this default
+define STORAGE_CREDENTIAL_NAME='adls_cred'

sql/credential-create.sql

@@ -0,0 +1,69 @@
+ARGUMENT 1 DEFAULT 'ALL'
+
+define user_param=&1
+undefine 1
+
+prompt "Creating credential: &user_param"
+
+-- Get the config information
+@config.sql
+
+-- This procedure will recreate credentials. You can specify a credential type (storage, OpenAI)
+
+DECLARE
+    l_exists number := 0;
+    l_type varchar2(20) := nvl(upper('&user_param'),'ALL');
+BEGIN
+    -- Azure OpenAI
+    if l_type in ('OPENAI','ALL') then
+        -- Create your credential. Replace it if already exists
+        select COUNT(*)
+        into l_exists
+        from user_credentials    
+        where upper(credential_name)=upper('&AZURE_OPENAI_CREDENTIAL_NAME');
+
+        IF l_exists = 1 THEN
+            dbms_cloud.drop_credential (
+                credential_name => '&AZURE_OPENAI_CREDENTIAL_NAME'
+            );
+        END IF;
+
+        dbms_cloud.create_credential (                                                 
+            credential_name => '&AZURE_OPENAI_CREDENTIAL_NAME',                                            
+            username => 'AZURE_OPENAI',                                                 
+            password => '&AZURE_OPENAI_KEY'
+        );
+
+    END IF; -- Azure OpenAI
+
+    if l_type in ('STORAGE','ALL') then
+        -- Create your credential. Replace it if already exists
+        select COUNT(*)
+        into l_exists
+        from user_credentials    
+        where upper(credential_name)=upper('&STORAGE_CREDENTIAL_NAME');
+
+        IF l_exists = 1 THEN
+            dbms_cloud.drop_credential (
+                credential_name => '&STORAGE_CREDENTIAL_NAME'
+            );
+        END IF;
+                          
+        dbms_cloud.create_credential(
+            credential_name => '&STORAGE_CREDENTIAL_NAME',
+            username => '&STORAGE_ACCOUNT_NAME',
+            password => '&STORAGE_KEY'
+        );
+
+    END IF; -- Storage   
+END;
+/
+
+-- Review the credentials
+COLUMN credential_name FORMAT A40
+COLUMN username FORMAT A40
+
+select 
+    credential_name, 
+    username 
+from user_credentials;

sql/data-export-to-datalake.sql

@@ -11,20 +11,13 @@
 @config.sql
 
 -- Create a credential in order to connect to data lake storage
-BEGIN
-    DBMS_CLOUD.CREATE_CREDENTIAL(
-        credential_name => 'ADLS_CRED',
-        username => '&STORAGE_ACCOUNT_NAME',
-        password => '&STORAGE_KEY'
-    );
-END;
-/
+@credential-create.sql storage
 
 -- List the files in that storage container
 SELECT object_name, bytes 
 FROM dbms_cloud.list_objects(
-    credential_name => 'ADLS_CRED',
-    location_uri => '&STORAGE_URL'
+    credential_name => '&STORAGE_CREDENTIAL_NAME',
+    location_uri => '&STORAGE_URL/adb-sample'
 );
 
 --
@@ -37,8 +30,8 @@ FROM genre;
 -- Export in CSV format
 BEGIN
   DBMS_CLOUD.EXPORT_DATA(
-    credential_name => 'ADLS_CRED',
-    file_uri_list => '&STORAGE_URL/data/genre/genre',
+    credential_name => '&STORAGE_CREDENTIAL_NAME',
+    file_uri_list => '&STORAGE_URL/adb-sample/data/genre/genre',
     query => 'SELECT * FROM genre',
     format => JSON_OBJECT('type' VALUE 'csv', 'delimiter' VALUE ',')
   );
@@ -48,6 +41,6 @@ END;
 -- List the files in that storage container. Notice the new genre data.
 SELECT object_name, bytes 
 FROM dbms_cloud.list_objects(
-    credential_name => 'ADLS_CRED',
-    location_uri => '&STORAGE_URL'
+    credential_name => '&STORAGE_CREDENTIAL_NAME',
+    location_uri => '&STORAGE_URL/adb-sample'
 );

sql/data-import-from-datalake.sql

@@ -6,20 +6,13 @@
 @config.sql
 
 -- Create a credential in order to connect to data lake storage
-BEGIN
-    DBMS_CLOUD.CREATE_CREDENTIAL(
-        credential_name => 'ADLS_CRED',
-        username => '&STORAGE_ACCOUNT_NAME',
-        password => '&STORAGE_KEY'
-    );
-END;
-/
+@credential-create.sql storage
 
 -- List the files
 SELECT object_name, bytes 
 FROM dbms_cloud.list_objects(
-    credential_name => 'ADLS_CRED',
-    location_uri => '&STORAGE_URL'
+    credential_name => '&STORAGE_CREDENTIAL_NAME',
+    location_uri => '&STORAGE_URL/adb-sample'
 );
 
 -- Create a table for movies that were created from books
@@ -37,7 +30,7 @@ CREATE TABLE movie_from_book
 BEGIN
   DBMS_CLOUD.COPY_DATA
   ( table_name        => 'MOVIE_FROM_BOOK',
-    credential_name   => 'ADLS_CRED',
+    credential_name   => '&STORAGE_CREDENTIAL_NAME',
     file_uri_list     => 'https://storagemartygubaradb.blob.core.windows.net/adb-sample/data/movie_from_book/*.csv',
     field_list        => 'MOVIE_NAME           CHAR(4000),
                           MOVIE_RELEASE_DATE   CHAR date_format DATE MASK "YYYY-MM-DD",

sql/select-ai-create-profile.sql

@@ -7,33 +7,9 @@
 
 -- config.sql contains the endpoints, resource groups and other settings required to connect to your Azure OpenAI deployment
 @config.sql
-
 
 -- Create a credential that allows the user to access the Azure OpenAI endpoint
-DECLARE
-    l_exists number := 0;
-BEGIN
-    -- Create your credential. Replace it if already exists
-    select COUNT(*)
-    into l_exists
-    from user_credentials    
-    where upper(credential_name)=upper('&AZURE_OPENAI_CREDENTIAL_NAME');
-
-    IF l_exists = 1 THEN
-        dbms_cloud.drop_credential (
-            credential_name => '&AZURE_OPENAI_CREDENTIAL_NAME'
-        );
-    END IF;
-
-    
-    dbms_cloud.create_credential (                                                 
-        credential_name => '&AZURE_OPENAI_CREDENTIAL_NAME',                                            
-        username => 'AZURE_OPENAI',                                                 
-        password => '&AZURE_OPENAI_KEY'
-    );
-
-END;                                                                           
-/  
+@credential-create.sql openai
 
 /*
   A Select AI profile describes the LLM you will use plus information that will be used for natural language queries. You can create as many